From 6cb9c8c339d50b064d0801c724e828fc4b3716f6 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 16 Oct 2026 08:11:23 +0000 Subject: [PATCH 01/47] [FS-2621] Add the materialization set as a direct-engine bundle resource A materialization set groups the materialized features that are created, updated and deleted together, so a bundle can declare a feature's offline and online materializations as one resource and reconcile them against their desired membership. Members are addressed by content rather than position: KeyedSlices pairs a configured member with a stored one the way UpdateMaterializationSet does, by feature name, online store and whether the member is streaming. Because a whole-member change is a shorter path than the per-field rules under materialized_features[*], and the matcher truncates a pattern longer than the path, such a change would otherwise classify as recreate and destroy every sibling the config left alone; OverrideChangeDesc keeps it on the update path. This branch holds only the hand-written half of the work. It does not compile on its own: ml.MaterializationSet reaches the SDK when the proto stages are promoted to PRIVATE_PREVIEW and a release carries them. Every generated artifact (cli.json, command stubs, *.generated.yml, refschema, jsonschema) is deliberately absent and is re-derived at that point. Co-authored-by: Isaac --- .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 7 ++ .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 3 + .../config/resources/materialization_set.go | 65 ++++++++++ bundle/config/resources_test.go | 13 ++ bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/all_test.go | 24 ++++ .../configs/materialization_sets.yml | 32 +++++ .../direct/dresources/materialization_set.go | 116 ++++++++++++++++++ .../dresources/materialization_set_test.go | 63 ++++++++++ .../direct/keyed_member_pattern_match_test.go | 29 +++++ bundle/phases/deploy.go | 1 + bundle/phases/messages.go | 5 + bundle/statemgmt/state_load_test.go | 19 +++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 14 +++ libs/testserver/materialization_set.go | 109 ++++++++++++++++ 18 files changed, 506 insertions(+) create mode 100644 bundle/config/resources/materialization_set.go create mode 100644 bundle/direct/dresources/configs/materialization_sets.yml create mode 100644 bundle/direct/dresources/materialization_set.go create mode 100644 bundle/direct/dresources/materialization_set_test.go create mode 100644 bundle/direct/keyed_member_pattern_match_test.go create mode 100644 libs/testserver/materialization_set.go diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 4ead9785e06..3b3eef2fe64 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -24,6 +24,7 @@ var unsupportedResources = []string{ "volumes", "schemas", "features", + "materialization_sets", "quality_monitors", "registered_models", "model_services", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index e9ea418d1d6..fd5f61bc111 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -159,6 +159,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Features: map[string]*resources.Feature{ "feature1": {Feature: ml.Feature{FullName: "catalog1.schema1.feature1"}}, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "materialization_set1": {MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "catalog1.schema1.feature1"}, + }, + }}, + }, Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 200939f2dd0..15b303bd7c7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -54,6 +54,7 @@ func allResourceTypes(t *testing.T) []string { "internal_immutable_snapshots", "job_runs", "jobs", + "materialization_sets", "mcp_services", "model_provider_services", "model_services", @@ -188,6 +189,7 @@ var allowList = []string{ "external_locations", "synced_database_tables", "jobs", + "materialization_sets", "pipelines", "models", "model_services", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 75fef099133..77b56bb6144 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -17,6 +17,7 @@ type Resources struct { Pipelines map[string]*resources.Pipeline `json:"pipelines,omitempty"` Features map[string]*resources.Feature `json:"features,omitempty"` + MaterializationSets map[string]*resources.MaterializationSet `json:"materialization_sets,omitempty"` Models map[string]*resources.MlflowModel `json:"models,omitempty"` Experiments map[string]*resources.MlflowExperiment `json:"experiments,omitempty"` ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` @@ -116,6 +117,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["models"], r.Models), collectResourceMap(descriptions["experiments"], r.Experiments), collectResourceMap(descriptions["features"], r.Features), + collectResourceMap(descriptions["materialization_sets"], r.MaterializationSets), collectResourceMap(descriptions["model_serving_endpoints"], r.ModelServingEndpoints), collectResourceMap(descriptions["model_services"], r.ModelServices), collectResourceMap(descriptions["mcp_services"], r.McpServices), @@ -203,6 +205,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "models": (&resources.MlflowModel{}).ResourceDescription(), "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), "features": (&resources.Feature{}).ResourceDescription(), + "materialization_sets": (&resources.MaterializationSet{}).ResourceDescription(), "instance_pools": (&resources.InstancePool{}).ResourceDescription(), "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), "model_services": (&resources.ModelService{}).ResourceDescription(), diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go new file mode 100644 index 00000000000..147b9541386 --- /dev/null +++ b/bundle/config/resources/materialization_set.go @@ -0,0 +1,65 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// MaterializationSet is the bundle config for a group of materialized features created and +// managed as one unit. Members are addressed by content rather than by position, so the set +// is reconciled against its desired membership rather than updated field by field. +type MaterializationSet struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + ml.MaterializationSet +} + +func (s *MaterializationSet) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s MaterializationSet) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *MaterializationSet) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.FeatureEngineering.GetMaterializationSet(ctx, ml.GetMaterializationSetRequest{ + MaterializationSetId: id, + }) + if err != nil { + log.Debugf(ctx, "materialization set %s does not exist: %v", id, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (*MaterializationSet) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "materialization_set", + PluralName: "materialization_sets", + SingularTitle: "Materialization Set", + PluralTitle: "Materialization Sets", + } +} + +// GetName returns the in-product name, which is empty: a set is identified only by its +// server-assigned id. +func (s *MaterializationSet) GetName() string { + return "" +} + +// InitializeURL leaves URL unset because a set has no standalone page in the workspace; its +// members surface under their own features. +func (s *MaterializationSet) InitializeURL(_ url.URL) {} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 963e7efd4ff..4cdafc52747 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -128,6 +128,9 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // Resources that intentionally have no workspace URL. noURL := map[string]bool{ "external_locations": true, + // A materialization set has no standalone workspace page; its members + // surface under their own features. + "materialization_sets": true, // A job run does have a workspace URL, but it's addressed by two IDs // (job + run) so it can't be expressed as a single-ID pattern here; it's // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. @@ -368,6 +371,15 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "my_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.myschema.my_feature"}, + }, + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -399,6 +411,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPipelinesAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockFeatureEngineeringAPI().EXPECT().GetFeature(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockFeatureEngineeringAPI().EXPECT().GetMaterializationSet(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockRegisteredModelsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 88f08048235..b13465550ca 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -12,6 +12,7 @@ var SupportedResources = map[string]any{ "pipelines": (*ResourcePipeline)(nil), "experiments": (*ResourceExperiment)(nil), "features": (*ResourceFeature)(nil), + "materialization_sets": (*ResourceMaterializationSet)(nil), "catalogs": (*ResourceCatalog)(nil), "schemas": (*ResourceSchema)(nil), "external_locations": (*ResourceExternalLocation)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 1b0da0602a2..c94a035d981 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -123,6 +123,30 @@ var testConfig map[string]any = map[string]any{ TimeseriesColumn: &ml.TimeseriesColumn{Name: "ts"}, }, + // A feature materialized to both an offline table and one online store: the two members + // share a feature name, so they exercise the member key's online-store discriminator. + "materialization_sets": &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + { + FeatureName: "main.default.my_feature", + CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "main", + SchemaName: "default", + }, + }, + { + FeatureName: "main.default.my_feature", + CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, + OnlineStoreConfig: &ml.OnlineStoreConfig{ + CatalogName: "main", + SchemaName: "default", + OnlineStoreName: "my_store", + }, + }, + }, + }, + "experiments": &resources.MlflowExperiment{ Name: "my-experiment", Tags: []ml.ExperimentTag{ diff --git a/bundle/direct/dresources/configs/materialization_sets.yml b/bundle/direct/dresources/configs/materialization_sets.yml new file mode 100644 index 00000000000..c85eb100d63 --- /dev/null +++ b/bundle/direct/dresources/configs/materialization_sets.yml @@ -0,0 +1,32 @@ +# UpdateMaterializationSet accepts only the `materialized_features` mask, and a member's own +# fields are write-once: the server has no way to alter one in place. The spec leaves them +# OPTIONAL rather than IMMUTABLE because UpdateMaterializedFeature can still change some of them +# on a member addressed directly, so materialization_sets.generated.yml will never cover them and +# these entries are permanent. +# +# A member added or removed whole also matches these patterns, because the matcher truncates a +# pattern longer than the path. ResourceMaterializationSet.OverrideChangeDesc downgrades that case +# to update so reconciling membership does not destroy the siblings the config left alone. +recreate_on_changes: + - field: materialized_features[*].feature_name + reason: immutable + - field: materialized_features[*].cron_schedule + reason: immutable + - field: materialized_features[*].cron_schedule_trigger + reason: immutable + - field: materialized_features[*].table_trigger + reason: immutable + - field: materialized_features[*].streaming_mode + reason: immutable + - field: materialized_features[*].tags + reason: immutable + - field: materialized_features[*].budget_policy_id + reason: immutable + - field: materialized_features[*].notifications + reason: immutable + +backend_defaults: + # The server sets a member's schedule state on create and writes it on only one node per + # feature, so a member the config leaves unset reports a value the bundle never wrote. Without + # this the undeclared case diffs as old=nil, new=nil, remote=ACTIVE and re-plans on every deploy. + - field: materialized_features[*].pipeline_schedule_state diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go new file mode 100644 index 00000000000..1b1e5ce60ed --- /dev/null +++ b/bundle/direct/dresources/materialization_set.go @@ -0,0 +1,116 @@ +package dresources + +import ( + "context" + "strconv" + "strings" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// https://docs.databricks.com/api/workspace/featureengineering +type ResourceMaterializationSet struct { + client *databricks.WorkspaceClient +} + +func (*ResourceMaterializationSet) New(client *databricks.WorkspaceClient) *ResourceMaterializationSet { + return &ResourceMaterializationSet{client: client} +} + +func (*ResourceMaterializationSet) PrepareState(input *resources.MaterializationSet) *ml.MaterializationSet { + return &input.MaterializationSet +} + +// materializationSetUpdatableFields is the only mask the update endpoint accepts, so "*" cannot +// be used. Membership is reconciled whole: the server deletes members the request omits. +var materializationSetUpdatableFields = []string{"materialized_features"} + +// escapeMemberKeyPart percent-escapes the separator so the join in materializationSetMemberKey is +// injective. A feature name is validated only for length and non-emptiness, so it may contain the +// separator; two members sharing a key would be diffed as one, and the request would then omit a +// member the server deletes because it cannot see it. +func escapeMemberKeyPart(part string) string { + part = strings.ReplaceAll(part, "%", "%25") + return strings.ReplaceAll(part, "|", "%7C") +} + +// materializationSetMemberKey identifies a member by content rather than position, matching how +// UpdateMaterializationSet pairs a requested member with a stored one. The parts are the feature +// name, the online store name (empty for the offline member, since an online destination always +// carries one), and whether the member is streaming. Feature name alone is not unique: one feature +// has an offline member plus one member per online store. +func materializationSetMemberKey(member ml.MaterializedFeature) (string, string) { + onlineStoreName := "" + if member.OnlineStoreConfig != nil { + onlineStoreName = member.OnlineStoreConfig.OnlineStoreName + } + return "member", strings.Join([]string{ + escapeMemberKeyPart(member.FeatureName), + escapeMemberKeyPart(onlineStoreName), + strconv.FormatBool(member.StreamingMode != nil), + }, "|") +} + +func (*ResourceMaterializationSet) KeyedSlices() map[string]any { + return map[string]any{ + "materialized_features": materializationSetMemberKey, + } +} + +var materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") + +// OverrideChangeDesc keeps a member added or removed whole on the update path. Such a change is a +// two-segment path, which still matches every per-field rule under materialized_features[*] because +// the matcher truncates a pattern longer than the path, so without this it classifies as recreate. +// +// Both directions belong on the update path, for different reasons. A removal is what the endpoint +// is for: it deletes the members the request omits, leaving the rest untouched. An addition the +// endpoint rejects outright, and routing it here surfaces that rejection naming the feature, +// instead of recreating the set and destroying every sibling the config left alone to achieve it. +// +// A one-segment path is deliberately left alone: it means one side has no members at all, which +// materialized_features being required makes invalid config, and recreate is the safe plan for it. +func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { + if path.Len() == 2 && path.HasPatternPrefix(materializedFeaturesPattern) { + change.Action = deployplan.Update + } + return nil +} + +func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml.MaterializationSet, error) { + return r.client.FeatureEngineering.GetMaterializationSet(ctx, ml.GetMaterializationSetRequest{ + MaterializationSetId: id, + }) +} + +func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { + response, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ + MaterializationSet: *config, + }) + if err != nil { + return "", nil, err + } + return response.MaterializationSetId, response, nil +} + +func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, config *ml.MaterializationSet, _ *PlanEntry) (*ml.MaterializationSet, error) { + update := *config + update.MaterializationSetId = id + + return r.client.FeatureEngineering.UpdateMaterializationSet(ctx, ml.UpdateMaterializationSetRequest{ + MaterializationSetId: id, + MaterializationSet: update, + UpdateMask: *fieldmask.New(materializationSetUpdatableFields), + }) +} + +func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ *ml.MaterializationSet) error { + return r.client.FeatureEngineering.DeleteMaterializationSet(ctx, ml.DeleteMaterializationSetRequest{ + MaterializationSetId: id, + }) +} diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go new file mode 100644 index 00000000000..02d92cfe32b --- /dev/null +++ b/bundle/direct/dresources/materialization_set_test.go @@ -0,0 +1,63 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" +) + +func TestMaterializationSetMemberKey(t *testing.T) { + offline := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + } + online := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "store_one"}, + } + streaming := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + StreamingMode: &ml.StreamingMode{}, + } + + field, value := materializationSetMemberKey(offline) + assert.Equal(t, "member", field) + assert.Equal(t, "main.default.feat||false", value) + + _, onlineValue := materializationSetMemberKey(online) + assert.Equal(t, "main.default.feat|store_one|false", onlineValue) + + _, streamingValue := materializationSetMemberKey(streaming) + assert.Equal(t, "main.default.feat||true", streamingValue) + + // The three members of one feature must not collide with each other. + assert.NotEqual(t, value, onlineValue) + assert.NotEqual(t, value, streamingValue) +} + +// A feature name may contain the separator, so without escaping these two distinct members would +// share a key: the plan would diff them as one and the update request would omit the member the +// server then deletes. +func TestMaterializationSetMemberKeyEscapesSeparator(t *testing.T) { + nameHoldsSeparator := ml.MaterializedFeature{ + FeatureName: "main.default.a|b", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "c"}, + } + storeHoldsSeparator := ml.MaterializedFeature{ + FeatureName: "main.default.a", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "b|c"}, + } + + _, first := materializationSetMemberKey(nameHoldsSeparator) + _, second := materializationSetMemberKey(storeHoldsSeparator) + assert.NotEqual(t, first, second) + + // The escape character itself is escaped first, so a name cannot forge an escaped separator. + _, forged := materializationSetMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.a%7Cb", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "c"}, + }) + assert.NotEqual(t, first, forged) +} diff --git a/bundle/direct/keyed_member_pattern_match_test.go b/bundle/direct/keyed_member_pattern_match_test.go new file mode 100644 index 00000000000..ac5e15f1a92 --- /dev/null +++ b/bundle/direct/keyed_member_pattern_match_test.go @@ -0,0 +1,29 @@ +package direct + +import ( + "testing" + + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +// A whole-member change on a keyed slice is a two-segment path, so it matches any deeper +// per-field pattern through the short-path branch of matchesFieldRuleBidirectional. A +// per-field recreate rule therefore recreates the parent when a member is merely added or +// removed, which is why a keyed-slice resource needs a carve-out before classification. +func TestKeyedMemberChangeMatchesPerFieldPattern(t *testing.T) { + // configs/.yml: materialized_features[*].feature_name + pattern := structpath.NewPatternDotString(nil, "materialized_features") + pattern = structpath.NewPatternBracketStar(pattern) + perField := structpath.NewPatternDotString(pattern, "feature_name") + + // structdiff emits this for a member present on one side only. + member := structpath.NewDotString(nil, "materialized_features") + member = structpath.NewKeyValue(member, "feature_name", "main.ns.feat") + + assert.Equal(t, 3, perField.Len()) + assert.Equal(t, 2, member.Len()) + assert.False(t, member.HasPatternPrefix(perField), "member is shorter, so not a direct prefix match") + assert.True(t, matchesFieldRuleBidirectional(member, perField), + "whole-member change still matches the per-field rule via the truncated pattern") +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..d7206c68607 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -41,6 +41,7 @@ var deployApprovalGroups = []approvalGroup{ {group: "postgres_databases", message: deleteOrRecreatePostgresDatabaseMessage}, {group: "vector_search_indexes", message: deleteOrRecreateVectorSearchIndexMessage}, {group: "genie_spaces", message: deleteOrRecreateGenieSpaceMessage}, + {group: "materialization_sets", message: deleteOrRecreateMaterializationSetMessage}, } func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 03eba35d8cf..bca18c53ab7 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -49,6 +49,11 @@ index drops all upserted vectors. Both can be expensive to rebuild:` deleteOrRecreateGenieSpaceMessage = ` This action will result in the deletion or recreation of the following Genie spaces. The conversation history attached to a deleted space is permanently lost:` + + deleteOrRecreateMaterializationSetMessage = ` +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change:` ) // DataLossWarning is the warning shown when a non-interactive command is about diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 4ec77578d2b..d55633c14ff 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -32,6 +32,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.models.test_mlflow_model": {ID: "1"}, "resources.experiments.test_mlflow_experiment": {ID: "1"}, "resources.features.test_feature": {ID: "main.default.test_feature"}, + "resources.materialization_sets.test_materialization_set": {ID: "set1"}, "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, "resources.model_services.test_model_service": {ID: "main.default.test_model_service"}, "resources.mcp_services.test_mcp_service": {ID: "main.default.test_mcp_service"}, @@ -225,6 +226,15 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "test_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.default.test_feature"}, + }, + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ @@ -664,6 +674,15 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "test_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.default.test_feature"}, + }, + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 78234c5ebfa..26362384bd8 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -229,6 +229,7 @@ type FakeWorkspace struct { ModelProviderServices map[string]catalog.ModelProviderService Features map[string]ml.Feature + MaterializationSets map[string]ml.MaterializationSet ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo VectorSearchIndexes map[string]fakeVectorSearchIndex @@ -513,6 +514,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { McpServices: map[string]catalog.McpService{}, ModelProviderServices: map[string]catalog.ModelProviderService{}, Features: map[string]ml.Feature{}, + MaterializationSets: map[string]ml.MaterializationSet{}, Volumes: map[string]catalog.VolumeInfo{}, Dashboards: NewEventualMap[string, *fakeDashboard](eventualConsistency), PublishedDashboards: map[string]dashboards.PublishedDashboard{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e1a14b9800b..af6673abe3d 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -851,6 +851,20 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.Features, req.Vars["full_name"]) }) + // Feature Engineering materialization sets: + server.Handle("POST", "/api/2.0/feature-engineering/materialization-sets", func(req Request) any { + return req.Workspace.MaterializationSetsCreate(req) + }) + server.Handle("GET", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.MaterializationSets, req.Vars["materialization_set_id"]) + }) + server.Handle("PATCH", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return req.Workspace.MaterializationSetsUpdate(req, req.Vars["materialization_set_id"]) + }) + server.Handle("DELETE", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return MapDelete(req.Workspace, req.Workspace.MaterializationSets, req.Vars["materialization_set_id"]) + }) + // Model Provider Services (AI Gateway): server.Handle("POST", "/api/2.1/unity-catalog/model-provider-services", func(req Request) any { diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go new file mode 100644 index 00000000000..9f5ced34689 --- /dev/null +++ b/libs/testserver/materialization_set.go @@ -0,0 +1,109 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// materializationSetMemberKey matches a requested member to a stored one the way the backend +// does: by feature name, online store and whether the member is streaming. Feature name alone is +// not unique because one feature has an offline member plus one member per online store. +func materializationSetMemberKey(member ml.MaterializedFeature) string { + onlineStoreName := "" + if member.OnlineStoreConfig != nil { + onlineStoreName = member.OnlineStoreConfig.OnlineStoreName + } + return fmt.Sprintf("%s|%s|%t", member.FeatureName, onlineStoreName, member.StreamingMode != nil) +} + +// populateMaterializationSetMember fills the fields the backend assigns on create: the member id, +// the online flag derived from the destination, and the schedule state the pipeline starts in. +func populateMaterializationSetMember(member ml.MaterializedFeature, setID string) ml.MaterializedFeature { + member.MaterializedFeatureId = nextUUID() + member.MaterializationSetId = setID + member.IsOnline = member.OnlineStoreConfig != nil + member.PipelineScheduleState = ml.MaterializedFeaturePipelineScheduleStateActive + return member +} + +// MaterializationSetsCreate fakes POST /api/2.0/feature-engineering/materialization-sets. +func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { + defer s.LockUnlock()() + + var set ml.MaterializationSet + if err := json.Unmarshal(req.Body, &set); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + if len(set.MaterializedFeatures) == 0 { + return Response{ + StatusCode: http.StatusBadRequest, + Body: "materialized_features must not be empty", + } + } + + set.MaterializationSetId = nextUUID() + for i, member := range set.MaterializedFeatures { + set.MaterializedFeatures[i] = populateMaterializationSetMember(member, set.MaterializationSetId) + } + + s.MaterializationSets[set.MaterializationSetId] = set + return Response{Body: set} +} + +// MaterializationSetsUpdate fakes PATCH +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. Membership is +// reconciled against the request: a member the request omits is deleted, and one it adds is +// created. A member matched to a stored one keeps that member's server-assigned fields, because +// the endpoint cannot alter a member in place. +func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Response { + defer s.LockUnlock()() + + existing, ok := s.MaterializationSets[setID] + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: fmt.Sprintf("materialization set %s not found", setID), + } + } + + var requested ml.MaterializationSet + if err := json.Unmarshal(req.Body, &requested); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + storedByKey := map[string]ml.MaterializedFeature{} + for _, member := range existing.MaterializedFeatures { + storedByKey[materializationSetMemberKey(member)] = member + } + + desired := requested.MaterializedFeatures + if len(desired) == 0 { + return Response{ + StatusCode: http.StatusBadRequest, + Body: "materialized_features must not be empty", + } + } + + members := make([]ml.MaterializedFeature, 0, len(desired)) + for _, member := range desired { + if stored, found := storedByKey[materializationSetMemberKey(member)]; found { + members = append(members, stored) + continue + } + members = append(members, populateMaterializationSetMember(member, setID)) + } + + existing.MaterializedFeatures = members + s.MaterializationSets[setID] = existing + return Response{Body: existing} +} From 6d3beba8f66009ca38a3a7c7fb50ef753859f6b4 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 16 Oct 2026 22:16:21 +0000 Subject: [PATCH 02/47] [FS-2621] Tighten the member-key and change-override doc comments Co-authored-by: Isaac --- .../direct/dresources/materialization_set.go | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 1b1e5ce60ed..f49c2329e80 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -39,11 +39,9 @@ func escapeMemberKeyPart(part string) string { return strings.ReplaceAll(part, "|", "%7C") } -// materializationSetMemberKey identifies a member by content rather than position, matching how -// UpdateMaterializationSet pairs a requested member with a stored one. The parts are the feature -// name, the online store name (empty for the offline member, since an online destination always -// carries one), and whether the member is streaming. Feature name alone is not unique: one feature -// has an offline member plus one member per online store. +// materializationSetMemberKey identifies a materialized feature member by content rather than +// position. The key components are the feature name, the online store name (empty for the offline +// member), and whether the trigger is streaming. func materializationSetMemberKey(member ml.MaterializedFeature) (string, string) { onlineStoreName := "" if member.OnlineStoreConfig != nil { @@ -64,17 +62,12 @@ func (*ResourceMaterializationSet) KeyedSlices() map[string]any { var materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") -// OverrideChangeDesc keeps a member added or removed whole on the update path. Such a change is a -// two-segment path, which still matches every per-field rule under materialized_features[*] because -// the matcher truncates a pattern longer than the path, so without this it classifies as recreate. -// -// Both directions belong on the update path, for different reasons. A removal is what the endpoint -// is for: it deletes the members the request omits, leaving the rest untouched. An addition the -// endpoint rejects outright, and routing it here surfaces that rejection naming the feature, -// instead of recreating the set and destroying every sibling the config left alone to achieve it. -// -// A one-segment path is deliberately left alone: it means one side has no members at all, which -// materialized_features being required makes invalid config, and recreate is the safe plan for it. +// OverrideChangeDesc marks a materialized feature member added to or removed from the set as an +// update. Such a change is a two-segment path, +// materialized_features[member='||'], which an +// immutable field rule under materialized_features[*] still matches. This counts that match as an +// update rather than a destroy-and-recreate, because a whole-member addition or removal is what the +// update endpoint handles. func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { if path.Len() == 2 && path.HasPatternPrefix(materializedFeaturesPattern) { change.Action = deployplan.Update From f90f4ddee94dc2ddbaf9f92b37a8c3085b69fff1 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 19:44:34 +0000 Subject: [PATCH 03/47] [FS-2621] Suppress the drift from destination fields the read never returns A member's destination fields are input-only: the server accepts them on write and strips them from every read. The generated spec:input_only rule under ignore_remote_changes exists to absorb that, but it cannot fire here. It only fires when the persisted value equals the configured one, and the planner recovers the persisted value by resolving the change path against stored state -- which means matching this resource's composite key label against a real string field of the element, so the lookup finds nothing. With no persisted value the rule is skipped and the change falls through to the immutable-field rule, re-planning every set with an online member as a recreate on every deploy. OverrideChangeDesc is the one place that needs no persisted value, so the suppression goes there. It is gated on the persisted value being absent rather than merely empty: a repoint the config actually asked for carries that value in from the local diff, and so still recreates, which is the right plan for a destination the server cannot alter in place. The fake now redacts the same fields on store. Echoing them back hid this entirely -- a local read matched the config where a real one does not, so the acceptance goldens agreed with a server that behaves differently. Co-authored-by: Isaac --- .../direct/dresources/materialization_set.go | 60 ++++- libs/testserver/materialization_set.go | 247 ++++++++++++++++-- 2 files changed, 278 insertions(+), 29 deletions(-) diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index f49c2329e80..7f97d276dd1 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -60,7 +60,48 @@ func (*ResourceMaterializationSet) KeyedSlices() map[string]any { } } -var materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") +var ( + materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") + materializedFeatureMember = structpath.NewPatternBracketStar(materializedFeaturesPattern) + + // The destination fields the server accepts on write and never returns on read. The whole + // offline block is input-only; online_store_name is returned, so it is deliberately absent. + redactedDestinationPatterns = []*structpath.PatternNode{ + structpath.NewPatternDotString(materializedFeatureMember, "offline_store_config"), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "catalog_name", + ), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "schema_name", + ), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "table_name_prefix", + ), + } +) + +// isRedactedDestinationField reports whether path names a destination field the read never returns. +func isRedactedDestinationField(path *structpath.PathNode) bool { + for _, pattern := range redactedDestinationPatterns { + if path.HasPatternPrefix(pattern) { + return true + } + } + return false +} + +// isAbsentRemotely reports whether a read returned nothing for a field, which for a redacted field +// is indistinguishable from the server having no value. +func isAbsentRemotely(remote any) bool { + if remote == nil { + return true + } + text, isString := remote.(string) + return isString && text == "" +} // OverrideChangeDesc marks a materialized feature member added to or removed from the set as an // update. Such a change is a two-segment path, @@ -71,6 +112,23 @@ var materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materiali func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { if path.Len() == 2 && path.HasPatternPrefix(materializedFeaturesPattern) { change.Action = deployplan.Update + return nil + } + + // A destination field the read never returns diffs as config-set versus remote-empty, and the + // generated spec:input_only rule under ignore_remote_changes cannot suppress it: that rule + // fires only when the persisted value equals the configured one, and the planner recovers the + // persisted value by resolving the change path against stored state. Resolving a keyed-slice + // segment matches the key label against a real string field of the element, and this + // resource's key label is composite, so the lookup finds nothing and the persisted value reads + // as unset. + // + // An unset Old is what separates that unrecoverable case from a repoint the config actually + // asked for, which carries the persisted value in from the local diff and still recreates. + if isRedactedDestinationField(path) && change.Old == nil && change.New != nil && + isAbsentRemotely(change.Remote) { + change.Action = deployplan.Skip + change.Reason = "input_only_not_returned" } return nil } diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 9f5ced34689..091aeab792e 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -4,13 +4,29 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "github.com/databricks/databricks-sdk-go/service/ml" ) -// materializationSetMemberKey matches a requested member to a stored one the way the backend -// does: by feature name, online store and whether the member is streaming. Feature name alone is -// not unique because one feature has an offline member plus one member per online store. +// This fake mirrors UpdateMaterializationSet closely enough that one set of acceptance goldens +// serves both the local run and a run against a real workspace. Where it rejects, it returns the +// backend's own message text, so a golden recorded locally is a claim about the real API that the +// cloud run then checks. + +func materializationSetError(statusCode int, errorCode, message string) Response { + return Response{ + StatusCode: statusCode, + Body: map[string]string{ + "error_code": errorCode, + "message": message, + }, + } +} + +// materializationSetMemberKey matches a requested member to a stored one the way the backend does: +// by feature name, online store and whether the trigger is streaming. Feature name alone is not +// unique because one feature has an offline member plus one member per online store. func materializationSetMemberKey(member ml.MaterializedFeature) string { onlineStoreName := "" if member.OnlineStoreConfig != nil { @@ -19,16 +35,52 @@ func materializationSetMemberKey(member ml.MaterializedFeature) string { return fmt.Sprintf("%s|%s|%t", member.FeatureName, onlineStoreName, member.StreamingMode != nil) } -// populateMaterializationSetMember fills the fields the backend assigns on create: the member id, -// the online flag derived from the destination, and the schedule state the pipeline starts in. +// effectivePipelineScheduleState resolves the create-path default, so a request mixing unset and +// ACTIVE members is not read as a disagreement. +func effectivePipelineScheduleState( + state ml.MaterializedFeaturePipelineScheduleState, +) ml.MaterializedFeaturePipelineScheduleState { + if state == "" { + return ml.MaterializedFeaturePipelineScheduleStateActive + } + return state +} + +// populateMaterializationSetMember fills the fields the backend assigns on create, then redacts +// the INPUT_ONLY destination fields the way the response path does. Storing them redacted is what +// makes a read here match a read from a real workspace, which an echoing fake hides. func populateMaterializationSetMember(member ml.MaterializedFeature, setID string) ml.MaterializedFeature { member.MaterializedFeatureId = nextUUID() member.MaterializationSetId = setID member.IsOnline = member.OnlineStoreConfig != nil - member.PipelineScheduleState = ml.MaterializedFeaturePipelineScheduleStateActive + member.PipelineScheduleState = effectivePipelineScheduleState(member.PipelineScheduleState) + member.OfflineStoreConfig = nil + if member.OnlineStoreConfig != nil { + member.OnlineStoreConfig = &ml.OnlineStoreConfig{ + OnlineStoreName: member.OnlineStoreConfig.OnlineStoreName, + } + } return member } +// validateBatchPipelineScheduleStates mirrors the create-path rule that every batch member agrees +// on one effective schedule state. Streaming members never carry one, so they are excluded. +func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Response { + states := map[ml.MaterializedFeaturePipelineScheduleState]bool{} + for _, member := range members { + if member.StreamingMode != nil { + continue + } + states[effectivePipelineScheduleState(member.PipelineScheduleState)] = true + } + if len(states) > 1 { + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "All materialized features must have the same pipeline schedule state.") + return &response + } + return nil +} + // MaterializationSetsCreate fakes POST /api/2.0/feature-engineering/materialization-sets. func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { defer s.LockUnlock()() @@ -42,10 +94,11 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { } if len(set.MaterializedFeatures) == 0 { - return Response{ - StatusCode: http.StatusBadRequest, - Body: "materialized_features must not be empty", - } + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "materialized_features must not be empty") + } + if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { + return *bad } set.MaterializationSetId = nextUUID() @@ -57,20 +110,82 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { return Response{Body: set} } +// effectiveCronScheduleMode treats an unset mode as MANUAL, so spelling MANUAL out is no change. +func effectiveCronScheduleMode(mode ml.CronScheduleMode) ml.CronScheduleMode { + if mode == "" { + return ml.CronScheduleModeManual + } + return mode +} + +// isCronScheduleChangeRequested mirrors the backend's comparison. The mode is always compared, so +// dropping DERIVED reads as asking to stop deriving; a DERIVED schedule's expression and timezone +// belong to the service, so leaving them out asks for nothing. That is what keeps a sawtooth +// feature updatable, where comparing the field as a whole would reject every update after create. +func isCronScheduleChangeRequested(requested, existing *ml.CronSchedule) bool { + if requested == nil || existing == nil { + return false + } + requestedMode := effectiveCronScheduleMode(requested.Mode) + if requestedMode != effectiveCronScheduleMode(existing.Mode) { + return true + } + if requestedMode == ml.CronScheduleModeDerived { + if requested.CronExpression != "" && requested.CronExpression != existing.CronExpression { + return true + } + } else if requested.CronExpression != existing.CronExpression { + return true + } + return requested.TimezoneId != "" && requested.TimezoneId != existing.TimezoneId +} + +// materializationSetDeletionTargets groups each feature's members into the one a delete is applied +// to and the ones it cascades to. Deleting a feature's offline member deletes its online ones; a +// feature with no offline member is deleted through each online one. +func materializationSetDeletionTargets(members []ml.MaterializedFeature) [][]ml.MaterializedFeature { + byFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if _, seen := byFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + byFeature[member.FeatureName] = append(byFeature[member.FeatureName], member) + } + + var targets [][]ml.MaterializedFeature + for _, featureName := range featureOrder { + var offline, online []ml.MaterializedFeature + for _, member := range byFeature[featureName] { + if member.IsOnline { + online = append(online, member) + } else { + offline = append(offline, member) + } + } + if len(offline) == 0 { + for _, member := range online { + targets = append(targets, []ml.MaterializedFeature{member}) + } + continue + } + // Assumes one offline destination per feature, as the backend does. + targets = append(targets, append([]ml.MaterializedFeature{offline[0]}, online...)) + } + return targets +} + // MaterializationSetsUpdate fakes PATCH -// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. Membership is -// reconciled against the request: a member the request omits is deleted, and one it adds is -// created. A member matched to a stored one keeps that member's server-assigned fields, because -// the endpoint cannot alter a member in place. +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. Field updates are +// validated before any deletion, so the destructive half is never attempted on a set that could +// not be fully updated. func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Response { defer s.LockUnlock()() existing, ok := s.MaterializationSets[setID] if !ok { - return Response{ - StatusCode: http.StatusNotFound, - Body: fmt.Sprintf("materialization set %s not found", setID), - } + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Materialization set with ID '%s' does not exist.", setID)) } var requested ml.MaterializationSet @@ -81,26 +196,102 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res } } + if len(requested.MaterializedFeatures) == 0 { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Updating materialization set '%s' with no materialized features is not "+ + "supported. Delete the set instead.", setID)) + } + storedByKey := map[string]ml.MaterializedFeature{} for _, member := range existing.MaterializedFeatures { storedByKey[materializationSetMemberKey(member)] = member } - desired := requested.MaterializedFeatures - if len(desired) == 0 { - return Response{ - StatusCode: http.StatusBadRequest, - Body: "materialized_features must not be empty", + // Match each requested member to a stored one. A requested member matching nothing is an + // addition, which this endpoint does not support. + matched := make([]ml.MaterializedFeature, 0, len(requested.MaterializedFeatures)) + requestedKeys := map[string]bool{} + for _, member := range requested.MaterializedFeatures { + stored, found := storedByKey[materializationSetMemberKey(member)] + if !found { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Materialized features cannot be added to an existing materialization "+ + "set, and feature '%s' does not match any member of this set. Recreate the "+ + "set instead.", member.FeatureName)) + } + if isCronScheduleChangeRequested(member.CronScheduleTrigger, stored.CronScheduleTrigger) { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("The 'cron_schedule_trigger' of feature '%s' cannot be changed after "+ + "its materialized features are created. Recreate the materialization set "+ + "instead.", member.FeatureName)) } + requestedKeys[materializationSetMemberKey(member)] = true + matched = append(matched, stored) } - members := make([]ml.MaterializedFeature, 0, len(desired)) - for _, member := range desired { - if stored, found := storedByKey[materializationSetMemberKey(member)]; found { - members = append(members, stored) + // Apply the one mutable field. It is written on the member it is applied to and no other, and + // applying it to an online member is refused. + scheduleStateChanges := map[string]ml.MaterializedFeaturePipelineScheduleState{} + for _, member := range requested.MaterializedFeatures { + key := materializationSetMemberKey(member) + stored := storedByKey[key] + if member.PipelineScheduleState == "" || + member.PipelineScheduleState == stored.PipelineScheduleState { continue } - members = append(members, populateMaterializationSetMember(member, setID)) + if stored.IsOnline { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Directly updating pipeline_schedule_state is only supported for "+ + "offline materialized features. Cannot update pipeline_schedule_state for "+ + "online materialization of feature '%s'. If this feature has an offline "+ + "materialization, please update that instead.", member.FeatureName)) + } + if member.PipelineScheduleState != ml.MaterializedFeaturePipelineScheduleStateActive && + member.PipelineScheduleState != ml.MaterializedFeaturePipelineScheduleStatePaused { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Pipeline schedule state cannot be updated to '%s'. It must be ACTIVE "+ + "or PAUSED.", member.PipelineScheduleState)) + } + scheduleStateChanges[key] = member.PipelineScheduleState + } + + // Delete the members the request omits. Because a delete cascades from a feature's offline + // member to its online ones, a removal that would also reach a member the request still lists + // cannot be expressed and is refused before anything is written. + removedIDs := map[string]bool{} + for _, member := range existing.MaterializedFeatures { + if !requestedKeys[materializationSetMemberKey(member)] { + removedIDs[member.MaterializedFeatureId] = true + } + } + if len(removedIDs) > 0 { + for _, target := range materializationSetDeletionTargets(existing.MaterializedFeatures) { + var unrequested []string + requestedForChange := false + for _, member := range target { + if removedIDs[member.MaterializedFeatureId] { + requestedForChange = true + } else { + unrequested = append(unrequested, "'"+member.MaterializedFeatureId+"'") + } + } + // A target the caller asked nothing about is not in play. + if requestedForChange && len(unrequested) > 0 { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Changing one materialized feature of feature '%s' also changes "+ + "%s, which the request does not change. Apply the same change to every "+ + "materialized feature of the feature.", + target[0].FeatureName, strings.Join(unrequested, ", "))) + } + } + } + + members := make([]ml.MaterializedFeature, 0, len(matched)) + for _, member := range matched { + if state, changed := scheduleStateChanges[materializationSetMemberKey(member)]; changed { + member.PipelineScheduleState = state + } + members = append(members, member) } existing.MaterializedFeatures = members From ea7907567cb70875c4405fd91c1ddae39f7f4c19 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 19:44:53 +0000 Subject: [PATCH 04/47] [FS-2621] Add acceptance tests for the materialization set The unit tests prove the member-key trap exists and that the pattern matcher behaves as the override assumes; they do not prove the override fires in a real plan. These cases close that gap, and each one pins a claim about the API rather than about the fake: the fake mirrors UpdateMaterializationSet's reconcile semantics and returns the backend's own message text, so turning on Cloud = true checks every golden here against a workspace. Six cases: a create and a redeploy that plans nothing; membership reconciliation routed to update rather than recreate; the schedule state applying in place without cascading; a write-once field caught by the planner before the request is built; the delete cascade scoped per feature; and the destination fields the read never returns staying quiet while a repoint of one still recreates. README.md records what each case pins, the three fixture changes a cloud run needs, and the two areas still uncovered. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 55 +++++++ .../materialization_sets/basic/databricks.yml | 42 +++++ .../materialization_sets/basic/out.test.toml | 2 + .../materialization_sets/basic/output.txt | 98 +++++++++++ .../materialization_sets/basic/script | 21 +++ .../destination/databricks.yml | 24 +++ .../destination/out.test.toml | 2 + .../destination/output.txt | 101 ++++++++++++ .../materialization_sets/destination/script | 32 ++++ .../immutable/databricks.yml | 12 ++ .../immutable/out.test.toml | 2 + .../materialization_sets/immutable/output.txt | 47 ++++++ .../materialization_sets/immutable/script | 23 +++ .../membership/databricks.yml | 17 ++ .../membership/out.test.toml | 2 + .../membership/output.txt | 152 ++++++++++++++++++ .../materialization_sets/membership/script | 30 ++++ .../removal_scope/databricks.yml | 19 +++ .../removal_scope/out.test.toml | 2 + .../removal_scope/output.txt | 54 +++++++ .../materialization_sets/removal_scope/script | 20 +++ .../schedule_state/databricks.yml | 16 ++ .../schedule_state/out.test.toml | 2 + .../schedule_state/output.txt | 111 +++++++++++++ .../schedule_state/script | 18 +++ .../resources/materialization_sets/test.toml | 7 + 26 files changed, 911 insertions(+) create mode 100644 acceptance/bundle/resources/materialization_sets/README.md create mode 100644 acceptance/bundle/resources/materialization_sets/basic/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/basic/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/basic/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/basic/script create mode 100644 acceptance/bundle/resources/materialization_sets/destination/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/destination/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/destination/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/destination/script create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/script create mode 100644 acceptance/bundle/resources/materialization_sets/membership/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/membership/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/membership/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/membership/script create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/script create mode 100644 acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/schedule_state/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/schedule_state/script create mode 100644 acceptance/bundle/resources/materialization_sets/test.toml diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md new file mode 100644 index 00000000000..db1f157a67c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -0,0 +1,55 @@ +# Materialization set acceptance tests + +These run locally against `libs/testserver/materialization_set.go`, which mirrors +`UpdateMaterializationSet`'s reconcile semantics rather than stubbing them. That is deliberate: one +`output.txt` serves both the local run and a `Cloud = true` run, so every rejection message here is +a claim about the real API, and the cloud run is what checks it. + +## What each case pins + +| Case | Claim | +|---|---| +| `basic` | create, then a redeploy that plans nothing — the server-assigned schedule state and the output-only member fields must not read as drift | +| `membership` | removing a whole feature's members plans as `update`, not `recreate`; removing one member of a feature on its own is refused because the delete cascades | +| `schedule_state` | pausing the offline member is an in-place update and does not cascade; declaring the state on an online member is refused | +| `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | +| `removal_scope` | the delete cascade is scoped per feature: a feature contributing one member can have it removed alone, while removing one member of a feature contributing two is refused | +| `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates — the two are told apart by whether the persisted value is recoverable | + +## Running against a workspace + +Add to `test.toml`: + +```toml +Cloud = true +Env.DATABRICKS_LITESWAP_TARGET = "" +``` + +The env var takes the unit name only — `liteswap.Apply` prepends `testenv://liteswap/`. Passing the +full traffic id routes to base staging instead, where these endpoints do not exist. + +Three fixture changes are needed first, because the local names are only valid against the fake: + +1. **Catalog and schema.** Declare `catalogs` and `schemas` resources using the harness's + `UNIQUE_NAME` and reference them from the feature names, so parallel runs do not collide. +2. **Online store.** `online_store_name` must name a store that exists on the target workspace. +3. **Features must exist.** Declare the `features` resources the set members reference; the fake + does not check that a member's feature exists, and the real API does. + +The workspace also needs a declarative feature preview enabled — `validateAnyDeclarativePreviewEnabled` +rejects every create otherwise, with `declarativePreviewsNotEnabled`. + +## Not covered yet + +- **Sawtooth.** A sawtooth feature in a set must declare all three members itself — batch offline, + batch online and streaming online — each batch one carrying a `DERIVED` cron schedule with no + expression, and both online ones the same destination. Declaring only the offline-and-online pair + is **rejected at create**, not silently expanded: `handleCreateMaterializationSet` stamps a + materialization set id on every member before delegating, and the normalization that would expand + the pair is skipped whenever that id is set, deliberately, so that a member maps to exactly one + materialized feature and the set stays diffable. The rejection names all three materializations. + A sawtooth fixture needs a `Stream`, which is not a bundle resource; `danielb.fs2680` already has + usable ones, so the fixture should reference an existing stream rather than create one. +- **Partial failure.** One member failing does not stop the others, and the error carries what was + updated, deleted and failed. Reproducing it needs per-member failure injection the fake does not + have. diff --git a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml new file mode 100644 index 00000000000..4d1839e0d9f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml @@ -0,0 +1,42 @@ +bundle: + name: test-bundle + +resources: + features: + batchfeat: + full_name: main.myschema.batchfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts + + materialization_sets: + set1: + materialized_features: + # One feature with both destinations: the offline member a delete cascades from, and the + # online member it cascades to. + - feature_name: main.myschema.batchfeat + offline_store_config: {} + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.batchfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: batchfeat + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/basic/out.test.toml b/acceptance/bundle/resources/materialization_sets/basic/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/basic/output.txt b/acceptance/bundle/resources/materialization_sets/basic/output.txt new file mode 100644 index 00000000000..738e5d44b5f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/output.txt @@ -0,0 +1,98 @@ + +=== Summary before deploy +>>> [CLI] bundle summary -o json +{ + "modified_status": "created", + "materialized_features": [ + { + "feature_name": "main.myschema.batchfeat", + "is_online": null + }, + { + "feature_name": "main.myschema.batchfeat", + "is_online": null + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.batchfeat +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.batchfeat", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + } + }, + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.batchfeat", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "batchfeat" + } + } + ] + } +} + +=== The server assigned a set id and a member id per member +>>> [CLI] bundle summary -o json +{ + "id": "[UUID]" +} + +=== Redeploying with no config change must plan nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //materialization-sets + +=== The schedule state the server assigned must not read as drift +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.batchfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 2 deleted + +>>> print_requests.py //materialization-sets +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]" +} diff --git a/acceptance/bundle/resources/materialization_sets/basic/script b/acceptance/bundle/resources/materialization_sets/basic/script new file mode 100644 index 00000000000..19a53b49aea --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/script @@ -0,0 +1,21 @@ +title "Summary before deploy" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {modified_status, materialized_features: [.materialized_features[] | {feature_name, is_online}]}' + +trace $CLI bundle deploy +trace print_requests.py //materialization-sets + +title "The server assigned a set id and a member id per member" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {id}' + +title "Redeploying with no config change must plan nothing" +trace $CLI bundle deploy +trace print_requests.py //materialization-sets + +title "The schedule state the server assigned must not read as drift" +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //materialization-sets + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/destination/databricks.yml b/acceptance/bundle/resources/materialization_sets/destination/databricks.yml new file mode 100644 index 00000000000..b71b583e8c9 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.destfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.destfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/destination/out.test.toml b/acceptance/bundle/resources/materialization_sets/destination/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/destination/output.txt b/acceptance/bundle/resources/materialization_sets/destination/output.txt new file mode 100644 index 00000000000..6b5aca32954 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A destination the read never returns is not drift: the config holds a value the server +=== never echoes, and with no persisted value to compare it to the suppression lives in +=== OverrideChangeDesc +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "skip", + "store_config_changes": { + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.catalog_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat||false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} + +=== Repointing the online destination is a real change the config asked for, so it keeps its +=== persisted value and still recreates +>>> update_file.py databricks.yml catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on catalog_name: other + schema_name: myschema + table_name_prefix: destfeat_on + +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "recreate", + "store_config_changes": { + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.catalog_name": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat||false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} + +>>> update_file.py databricks.yml catalog_name: other catalog_name: main + +=== The offline block is input-only whole, and repointing it recreates the same way +>>> update_file.py databricks.yml table_name_prefix: destfeat_off table_name_prefix: destfeat_moved + +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "recreate", + "store_config_changes": { + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.catalog_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat||false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='main.myschema.destfeat||false'].offline_store_config.table_name_prefix": { + "action": "recreate", + "reason": "spec:immutable" + } + } +} diff --git a/acceptance/bundle/resources/materialization_sets/destination/script b/acceptance/bundle/resources/materialization_sets/destination/script new file mode 100644 index 00000000000..dde15565eb2 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/script @@ -0,0 +1,32 @@ +trace $CLI bundle deploy + +destination_changes() { + jq '.plan["resources.materialization_sets.set1"] + | {action, store_config_changes: (.changes + | with_entries(select(.key | test("store_config"))) + | map_values({action, reason}))}' out.plan.json +} + +title "A destination the read never returns is not drift: the config holds a value the server" +title "never echoes, and with no persisted value to compare it to the suppression lives in" +title "OverrideChangeDesc" +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes + +title "Repointing the online destination is a real change the config asked for, so it keeps its" +title "persisted value and still recreates" +trace update_file.py databricks.yml ' catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on' ' catalog_name: other + schema_name: myschema + table_name_prefix: destfeat_on' +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes +trace update_file.py databricks.yml ' catalog_name: other' ' catalog_name: main' + +title "The offline block is input-only whole, and repointing it recreates the same way" +trace update_file.py databricks.yml 'table_name_prefix: destfeat_off' 'table_name_prefix: destfeat_moved' +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml new file mode 100644 index 00000000000..51a64ca8ab2 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.featA + offline_store_config: {} + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt new file mode 100644 index 00000000000..235e5658d7c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A member field the endpoint cannot alter is caught by the planner as a recreate, so the +=== request never reaches the server +>>> update_file.py databricks.yml 0 0 * * * ? 0 30 * * * ? + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> update_file.py databricks.yml 0 30 * * * ? 0 0 * * * ? + +=== Adding a member is routed to update, where the server refuses it by name rather than +=== recreating the set and destroying the members the config left alone +>>> update_file.py databricks.yml cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.featB + offline_store_config: {} + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features cannot be added to an existing materialization set, and feature 'main.myschema.featB' does not match any member of this set. Recreate the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features cannot be added to an existing materialization set, and feature 'main.myschema.featB' does not match any member of this set. Recreate the set instead. + +Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script new file mode 100644 index 00000000000..d7bfea94358 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -0,0 +1,23 @@ +trace $CLI bundle deploy + +title "A member field the endpoint cannot alter is caught by the planner as a recreate, so the" +title "request never reaches the server" +trace update_file.py databricks.yml '0 0 * * * ?' '0 30 * * * ?' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace update_file.py databricks.yml '0 30 * * * ?' '0 0 * * * ?' + +title "Adding a member is routed to update, where the server refuses it by name rather than" +title "recreating the set and destroying the members the config left alone" +trace update_file.py databricks.yml ' cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?"' ' cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.featB + offline_store_config: {}' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace musterr $CLI bundle deploy + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml new file mode 100644 index 00000000000..b30b5cd5585 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.featA + offline_store_config: {} + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: feata + - feature_name: main.myschema.featB + offline_store_config: {} diff --git a/acceptance/bundle/resources/materialization_sets/membership/out.test.toml b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt new file mode 100644 index 00000000000..50c3420f532 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -0,0 +1,152 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "feature_name": "main.myschema.featA", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + } + }, + { + "feature_name": "main.myschema.featA", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "feata" + } + }, + { + "feature_name": "main.myschema.featB", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + } + } + ] + } +} + +=== Removing one member of a feature that has two is refused: the delete cascades from the +=== offline member to its online one, so it would also reach a member still declared +>>> update_file.py databricks.yml - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: feata + + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. + +Files: 1 uploaded, 0 deleted + +>>> print_requests.py //materialization-sets +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "update_mask": "materialized_features" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "feature_name": "main.myschema.featA", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + } + }, + { + "feature_name": "main.myschema.featB", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + } + } + ] + } +} + +=== Removing a whole feature's members plans as update, not recreate, and leaves the sibling +>>> update_file.py databricks.yml - feature_name: main.myschema.featA + offline_store_config: {} + + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //materialization-sets +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "update_mask": "materialized_features" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "feature_name": "main.myschema.featB", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + } + } + ] + } +} + +=== The surviving member kept its id, so the set was reconciled rather than rebuilt +>>> [CLI] bundle summary -o json +[ + { + "feature_name": "main.myschema.featB", + "is_online": null + } +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/materialization_sets/membership/script b/acceptance/bundle/resources/materialization_sets/membership/script new file mode 100644 index 00000000000..a8008fe6a6e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/script @@ -0,0 +1,30 @@ +trace $CLI bundle deploy +trace print_requests.py //materialization-sets + +title "Removing one member of a feature that has two is refused: the delete cascades from the" +title "offline member to its online one, so it would also reach a member still declared" +trace update_file.py databricks.yml ' - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: feata +' '' +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py //materialization-sets + +title "Removing a whole feature's members plans as update, not recreate, and leaves the sibling" +trace update_file.py databricks.yml ' - feature_name: main.myschema.featA + offline_store_config: {} +' '' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy --auto-approve +trace print_requests.py //materialization-sets + +title "The surviving member kept its id, so the set was reconciled rather than rebuilt" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' + +trace $CLI bundle destroy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml new file mode 100644 index 00000000000..a07f6574337 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + # F1 contributes one member, so nothing cascades from it. + - feature_name: main.myschema.featF1 + offline_store_config: {} + # F2 contributes both, so a delete on its offline member reaches its online one. + - feature_name: main.myschema.featF2 + offline_store_config: {} + - feature_name: main.myschema.featF2 + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: featf2 diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt new file mode 100644 index 00000000000..c6ca5ac3e3f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -0,0 +1,54 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Removing F1's only member is allowed: the cascade is scoped per feature, and F2's pair is +=== untouched because the request asks nothing about it +>>> update_file.py databricks.yml - feature_name: main.myschema.featF1 + offline_store_config: {} + + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle summary -o json +[ + { + "feature_name": "main.myschema.featF2", + "is_online": null + }, + { + "feature_name": "main.myschema.featF2", + "is_online": null + } +] + +=== Removing F2's offline member alone is refused, because that delete cascades to F2's online +=== member, which the request still declares +>>> update_file.py databricks.yml - feature_name: main.myschema.featF2 + offline_store_config: {} + + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.myschema.featF2' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Changing one materialized feature of feature 'main.myschema.featF2' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script new file mode 100644 index 00000000000..e08b7779fdb --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/script @@ -0,0 +1,20 @@ +trace $CLI bundle deploy + +title "Removing F1's only member is allowed: the cascade is scoped per feature, and F2's pair is" +title "untouched because the request asks nothing about it" +trace update_file.py databricks.yml ' - feature_name: main.myschema.featF1 + offline_store_config: {} +' '' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy --auto-approve +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' + +title "Removing F2's offline member alone is refused, because that delete cascades to F2's online" +title "member, which the request still declares" +trace update_file.py databricks.yml ' - feature_name: main.myschema.featF2 + offline_store_config: {} +' '' +trace musterr $CLI bundle deploy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml new file mode 100644 index 00000000000..711a7d5f016 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.featA + offline_store_config: {} + pipeline_schedule_state: ACTIVE + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: feata diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml b/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt new file mode 100644 index 00000000000..3f1893fd9e8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt @@ -0,0 +1,111 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Pausing the offline member is an in-place update +>>> update_file.py databricks.yml pipeline_schedule_state: ACTIVE pipeline_schedule_state: PAUSED + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "feature_name": "main.myschema.featA", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + }, + "pipeline_schedule_state": "ACTIVE" + }, + { + "feature_name": "main.myschema.featA", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "feata" + } + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "update_mask": "materialized_features" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "feature_name": "main.myschema.featA", + "offline_store_config": { + "catalog_name": "", + "schema_name": "", + "table_name_prefix": "" + }, + "pipeline_schedule_state": "PAUSED" + }, + { + "feature_name": "main.myschema.featA", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "feata" + } + } + ] + } +} + +=== The change did not cascade: the online member keeps its own state +>>> [CLI] bundle summary -o json +[ + { + "is_online": null, + "pipeline_schedule_state": "PAUSED" + }, + { + "is_online": null, + "pipeline_schedule_state": null + } +] + +=== Declaring the state on the online member is refused, since the write would land on it +>>> update_file.py databricks.yml table_name_prefix: feata table_name_prefix: feata + pipeline_schedule_state: PAUSED + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Directly updating pipeline_schedule_state is only supported for offline materialized features. Cannot update pipeline_schedule_state for online materialization of feature 'main.myschema.featA'. If this feature has an offline materialization, please update that instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Directly updating pipeline_schedule_state is only supported for offline materialized features. Cannot update pipeline_schedule_state for online materialization of feature 'main.myschema.featA'. If this feature has an offline materialization, please update that instead. + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/script b/acceptance/bundle/resources/materialization_sets/schedule_state/script new file mode 100644 index 00000000000..586fe83094e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/script @@ -0,0 +1,18 @@ +trace $CLI bundle deploy + +title "Pausing the offline member is an in-place update" +trace update_file.py databricks.yml 'pipeline_schedule_state: ACTIVE' 'pipeline_schedule_state: PAUSED' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy +trace print_requests.py //materialization-sets + +title "The change did not cascade: the online member keeps its own state" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' + +title "Declaring the state on the online member is refused, since the write would land on it" +trace update_file.py databricks.yml ' table_name_prefix: feata' ' table_name_prefix: feata + pipeline_schedule_state: PAUSED' +trace musterr $CLI bundle deploy + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/test.toml b/acceptance/bundle/resources/materialization_sets/test.toml new file mode 100644 index 00000000000..1b9a24bbf2f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/test.toml @@ -0,0 +1,7 @@ +# Local-only for now. The fake in libs/testserver/materialization_set.go mirrors +# UpdateMaterializationSet's reconcile semantics, so these goldens are a claim about the real API; +# turning on Cloud = true checks that claim against a workspace. See README.md for the delta. +Ignore = [ + "databricks.yml", + ".databricks", +] From 1246725f344008e0368ddf3089492c8eb357e3c7 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 20:30:43 +0000 Subject: [PATCH 05/47] [FS-2621] Say what the change override does, and why the generated rule cannot The doc comment described only the membership half and buried the reasoning for it, and the suppression comment said the persisted value "reads as unset" without saying that it is recorded in full and merely unreachable. Name both jobs on the function, then put each one's reasoning on the branch that does it. Co-authored-by: Isaac --- .../direct/dresources/materialization_set.go | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 7f97d276dd1..02bd611b611 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -103,28 +103,28 @@ func isAbsentRemotely(remote any) bool { return isString && text == "" } -// OverrideChangeDesc marks a materialized feature member added to or removed from the set as an -// update. Such a change is a two-segment path, -// materialized_features[member='||'], which an -// immutable field rule under materialized_features[*] still matches. This counts that match as an -// update rather than a destroy-and-recreate, because a whole-member addition or removal is what the -// update endpoint handles. +// OverrideChangeDesc does two things the generic classification gets wrong here: it marks a whole +// materialized feature member added to or removed from the set as an update rather than a +// destroy-and-recreate, and it suppresses the drift left behind by an ignore_remote_changes rule +// that cannot fire. func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { + // A member added to or removed from the set is a two-segment path, + // materialized_features[member='||'], which an + // immutable field rule under materialized_features[*] still matches. That match counts as an + // update rather than a destroy-and-recreate, because a whole-member addition or removal is what + // the update endpoint handles. if path.Len() == 2 && path.HasPatternPrefix(materializedFeaturesPattern) { change.Action = deployplan.Update return nil } - // A destination field the read never returns diffs as config-set versus remote-empty, and the - // generated spec:input_only rule under ignore_remote_changes cannot suppress it: that rule - // fires only when the persisted value equals the configured one, and the planner recovers the - // persisted value by resolving the change path against stored state. Resolving a keyed-slice - // segment matches the key label against a real string field of the element, and this - // resource's key label is composite, so the lookup finds nothing and the persisted value reads - // as unset. - // - // An unset Old is what separates that unrecoverable case from a repoint the config actually - // asked for, which carries the persisted value in from the local diff and still recreates. + // The ignore_remote_changes suppression on a destination field cannot fire for this resource, + // which keys its member slice on a composite of feature name, online store name and whether the + // member is streaming. The suppression fires only when the persisted value equals the configured + // one, and the persisted-value lookup resolves that composite against the element's field names, + // which never match, so it always returns nothing. An unset change.Old is the surrogate for the + // persisted value being unreachable; a repoint the config asked for carries that value in from + // the local diff instead, and so still recreates. if isRedactedDestinationField(path) && change.Old == nil && change.New != nil && isAbsentRemotely(change.Remote) { change.Action = deployplan.Skip From f527f2973060e92c91c5c3d5bce0163e015703e8 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 20:55:57 +0000 Subject: [PATCH 06/47] [FS-2621] Cover sawtooth and streaming, and correct one refusal against the server Neither shape was covered. Sawtooth is the one a user is most likely to get wrong, because the offline-and-online batch pair looks complete and is not: a set stamps its id on every member before the backend reaches the normalization that would expand the pair, and that normalization is skipped whenever the id is set, so the third materialization has to be declared. The fake now resolves a member's feature and applies the same three-member rule, which is what lets one golden carry the claim. Both new cases reference an existing Stream, since a Stream is not a bundle resource, and both reach the feature through ${resources.features.*.full_name} rather than repeating its name. That reference is load-bearing: without it the engine has no edge between the feature and the set, and the deploy order decides whether the set create sees a feature at all. Running the pair against a workspace also showed the online pipeline_schedule_state refusal had drifted. The server says the change "would be written on an online materialized feature" and returns BAD_REQUEST; the fake claimed a different sentence and INVALID_PARAMETER_VALUE, so schedule_state had been asserting a message no server produces. Corrected to the observed text, which moves that golden too. Co-authored-by: Isaac --- .../sawtooth/databricks.yml.tmpl | 51 ++++++++++++++ .../sawtooth/out.test.toml | 2 + .../materialization_sets/sawtooth/output.txt | 41 +++++++++++ .../materialization_sets/sawtooth/script | 43 ++++++++++++ .../materialization_sets/sawtooth/test.toml | 8 +++ .../sawtooth/twoleg.yml.tmpl | 42 +++++++++++ .../schedule_state/output.txt | 6 +- .../streaming/databricks.yml.tmpl | 36 ++++++++++ .../streaming/out.test.toml | 2 + .../materialization_sets/streaming/output.txt | 44 ++++++++++++ .../materialization_sets/streaming/script | 39 +++++++++++ .../materialization_sets/streaming/test.toml | 8 +++ libs/testserver/materialization_set.go | 70 +++++++++++++++++-- 13 files changed, 384 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/script create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/script create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/test.toml diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl new file mode 100644 index 00000000000..8f76bfee764 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl @@ -0,0 +1,51 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt new file mode 100644 index 00000000000..d8a814027f8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -0,0 +1,41 @@ + +=== A sawtooth feature declaring only the offline and online batch pair is refused at create. +=== The pair is not expanded into the third materialization: a set stamps its id on every +=== member before the backend reaches that normalization, and the normalization is skipped +=== whenever the id is set, so one member stays one materialized feature. +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Created features.sawfeat +Error: cannot create resources.materialization_sets.set1: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (400 BAD_REQUEST) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. + +Files: 3 uploaded, 0 deleted + +=== Declaring all three creates the set +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +=== A redeploy plans nothing. Both online members carry a destination the read never returns, +=== so this is also what proves the input-only suppression holds for a three-member set. +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== Every suppressed destination change, by reason +>>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length out.plan.json +7 diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/script b/acceptance/bundle/resources/materialization_sets/sawtooth/script new file mode 100644 index 00000000000..d44a778b64b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/script @@ -0,0 +1,43 @@ +# A sawtooth feature needs a Stream source, and a Stream is not a bundle resource, so this case +# references one that already exists rather than creating it. Everything else is per-run. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" + +# The harness resolves auth from the ambient environment, which on a cloud run is the workspace +# under test. Pointing a local run at a named profile needs it in the bundle, because +# DATABRICKS_CONFIG_PROFILE does not survive into the CLI subprocess. +pin_profile() { + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true +} +trap cleanup EXIT + +title "A sawtooth feature declaring only the offline and online batch pair is refused at create." +title "The pair is not expanded into the third materialization: a set stamps its id on every" +title "member before the backend reaches that normalization, and the normalization is skipped" +title "whenever the id is set, so one member stays one materialized feature." +envsubst < twoleg.yml.tmpl > databricks.yml +pin_profile +trace musterr $CLI bundle deploy + +title "Declaring all three creates the set" +envsubst < databricks.yml.tmpl > databricks.yml +pin_profile +trace $CLI bundle deploy + +title "A redeploy plans nothing. Both online members carry a destination the read never returns," +title "so this is also what proves the input-only suppression holds for a three-member set." +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy + +title "Every suppressed destination change, by reason" +trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length' out.plan.json + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml new file mode 100644 index 00000000000..45bb2a74f39 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml @@ -0,0 +1,8 @@ +# Pinned to a workspace holding a Stream, since a Stream is not a bundle resource. Override +# MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl new file mode 100644 index 00000000000..54b4be4fb96 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt index 3f1893fd9e8..a49d10120b5 100644 --- a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt @@ -101,11 +101,11 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Directly updating pipeline_schedule_state is only supported for offline materialized features. Cannot update pipeline_schedule_state for online materialization of feature 'main.myschema.featA'. If this feature has an offline materialization, please update that instead. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: The 'pipeline_schedule_state' of feature 'main.myschema.featA' cannot be changed, because the change would be written on an online materialized feature. (400 BAD_REQUEST) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Directly updating pipeline_schedule_state is only supported for offline materialized features. Cannot update pipeline_schedule_state for online materialization of feature 'main.myschema.featA'. If this feature has an offline materialization, please update that instead. +API error_code: BAD_REQUEST +API message: The 'pipeline_schedule_state' of feature 'main.myschema.featA' cannot be changed, because the change would be written on an online materialized feature. Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl new file mode 100644 index 00000000000..8ab5d8b17e3 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl @@ -0,0 +1,36 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml b/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/streaming/output.txt b/acceptance/bundle/resources/materialization_sets/streaming/output.txt new file mode 100644 index 00000000000..4bf6a953d61 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/output.txt @@ -0,0 +1,44 @@ + +=== A streaming member carries a trigger and no schedule, so none of the batch rules apply +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Created features.streamfeat +Created materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A redeploy plans nothing. The schedule state the server assigns to a lone streaming member +=== must not read as drift, and neither must its online destination. +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== Pausing a streaming member is refused: the schedule state applies to offline members only +>>> update_file.py databricks.yml streaming_mode: pipeline_schedule_state: PAUSED + streaming_mode: + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: The 'pipeline_schedule_state' of feature 'danielb.fs2680.stream_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. (400 BAD_REQUEST) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: The 'pipeline_schedule_state' of feature 'danielb.fs2680.stream_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. + +Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/streaming/script b/acceptance/bundle/resources/materialization_sets/streaming/script new file mode 100644 index 00000000000..80596f64035 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/script @@ -0,0 +1,39 @@ +# Like the sawtooth case, this references an existing Stream rather than creating one. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" + +# The harness resolves auth from the ambient environment, which on a cloud run is the workspace +# under test. Pointing a local run at a named profile needs it in the bundle, because +# DATABRICKS_CONFIG_PROFILE does not survive into the CLI subprocess. +pin_profile() { + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml +pin_profile + +title "A streaming member carries a trigger and no schedule, so none of the batch rules apply" +trace $CLI bundle deploy + +title "A redeploy plans nothing. The schedule state the server assigns to a lone streaming member" +title "must not read as drift, and neither must its online destination." +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy + +title "Pausing a streaming member is refused: the schedule state applies to offline members only" +trace update_file.py databricks.yml ' streaming_mode:' ' pipeline_schedule_state: PAUSED + streaming_mode:' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace musterr $CLI bundle deploy + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/streaming/test.toml b/acceptance/bundle/resources/materialization_sets/streaming/test.toml new file mode 100644 index 00000000000..45bb2a74f39 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/test.toml @@ -0,0 +1,8 @@ +# Pinned to a workspace holding a Stream, since a Stream is not a bundle resource. Override +# MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 091aeab792e..bcfc800d158 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -81,6 +81,64 @@ func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Resp return nil } +// sawtoothMaterializedFeatureCount is how many materialized features a sawtooth window produces: +// an offline batch one, an online batch one and an online streaming one. +const sawtoothMaterializedFeatureCount = 3 + +// isSawtoothFeature reports whether a stored feature aggregates over a sawtooth window. An unknown +// feature is not treated as sawtooth: the real create resolves the feature first and fails on a +// missing one, which is a separate rejection this fake does not model. +func (s *FakeWorkspace) isSawtoothFeature(featureName string) bool { + feature, stored := s.Features[featureName] + if !stored { + return false + } + aggregation := feature.Function.AggregationFunction + return aggregation != nil && aggregation.TimeWindow != nil && aggregation.TimeWindow.Sawtooth != nil +} + +// validateSawtoothMaterializedFeatures mirrors the rule that a sawtooth feature declares all three +// of its materialized features itself. The request is not expanded for it: a materialization set +// stamps a set id on every member before the backend reaches its normalization, and that +// normalization is skipped whenever the id is set, so a member maps to exactly one materialized +// feature and the set stays diffable. +func (s *FakeWorkspace) validateSawtoothMaterializedFeatures(members []ml.MaterializedFeature) *Response { + byFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if !s.isSawtoothFeature(member.FeatureName) { + continue + } + if _, seen := byFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + byFeature[member.FeatureName] = append(byFeature[member.FeatureName], member) + } + + for _, featureName := range featureOrder { + var offlineBatch, onlineBatch, onlineStreaming int + for _, member := range byFeature[featureName] { + switch { + case member.OnlineStoreConfig != nil && member.StreamingMode != nil: + onlineStreaming++ + case member.OnlineStoreConfig != nil: + onlineBatch++ + case member.OfflineStoreConfig != nil && member.StreamingMode == nil: + offlineBatch++ + } + } + if len(byFeature[featureName]) != sawtoothMaterializedFeatureCount || + offlineBatch != 1 || onlineBatch != 1 || onlineStreaming != 1 { + response := materializationSetError(http.StatusBadRequest, "BAD_REQUEST", fmt.Sprintf( + "Feature '%s' uses a sawtooth window, which requires an offline batch "+ + "materialization, an online batch materialization and an online streaming "+ + "materialization. Declare one materialize feature request for each.", featureName)) + return &response + } + } + return nil +} + // MaterializationSetsCreate fakes POST /api/2.0/feature-engineering/materialization-sets. func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { defer s.LockUnlock()() @@ -100,6 +158,9 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { return *bad } + if bad := s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures); bad != nil { + return *bad + } set.MaterializationSetId = nextUUID() for i, member := range set.MaterializedFeatures { @@ -240,11 +301,10 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res continue } if stored.IsOnline { - return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("Directly updating pipeline_schedule_state is only supported for "+ - "offline materialized features. Cannot update pipeline_schedule_state for "+ - "online materialization of feature '%s'. If this feature has an offline "+ - "materialization, please update that instead.", member.FeatureName)) + return materializationSetError(http.StatusBadRequest, "BAD_REQUEST", + fmt.Sprintf("The 'pipeline_schedule_state' of feature '%s' cannot be changed, "+ + "because the change would be written on an online materialized feature.", + member.FeatureName)) } if member.PipelineScheduleState != ml.MaterializedFeaturePipelineScheduleStateActive && member.PipelineScheduleState != ml.MaterializedFeaturePipelineScheduleStatePaused { From 50bb33ea4a02384581a9fa6d312a1b145e40be27 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 20:57:06 +0000 Subject: [PATCH 07/47] [FS-2621] Record the two new cases and what a local cloud run needs Sawtooth moves out of "not covered yet" and bind takes its place. Also writes down the two auth facts a local cloud run trips over, since neither is visible from the fixture: the profile environment variable does not reach the CLI subprocess, and the harness authenticates on its own before any script runs. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index db1f157a67c..76c7e4474a5 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -15,9 +15,20 @@ a claim about the real API, and the cloud run is what checks it. | `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | | `removal_scope` | the delete cascade is scoped per feature: a feature contributing one member can have it removed alone, while removing one member of a feature contributing two is refused | | `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates — the two are told apart by whether the persisted value is recoverable | +| `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing | +| `streaming` | a lone streaming online member needs no schedule and no cron, and pausing it is refused | ## Running against a workspace +`sawtooth` and `streaming` already carry `Cloud = true`. Both reference an existing `Stream`, +because a Stream is not a bundle resource; override `MATSET_CATALOG`, `MATSET_SCHEMA`, +`MATSET_ONLINE_STORE` and `MATSET_STREAM` to move them to another workspace, and set +`MATSET_PROFILE` to pin the bundle to a named profile. **`DATABRICKS_CONFIG_PROFILE` does not reach +the CLI subprocess**, and the harness builds its own workspace client from the ambient environment +before any script runs, so a local cloud run needs working ambient auth to the target workspace. + +### The other six cases + Add to `test.toml`: ```toml @@ -41,15 +52,8 @@ rejects every create otherwise, with `declarativePreviewsNotEnabled`. ## Not covered yet -- **Sawtooth.** A sawtooth feature in a set must declare all three members itself — batch offline, - batch online and streaming online — each batch one carrying a `DERIVED` cron schedule with no - expression, and both online ones the same destination. Declaring only the offline-and-online pair - is **rejected at create**, not silently expanded: `handleCreateMaterializationSet` stamps a - materialization set id on every member before delegating, and the normalization that would expand - the pair is skipped whenever that id is set, deliberately, so that a member maps to exactly one - materialized feature and the set stays diffable. The rejection names all three materializations. - A sawtooth fixture needs a `Stream`, which is not a bundle resource; `danielb.fs2680` already has - usable ones, so the fixture should reference an existing stream rather than create one. +- **Bind and unbind.** `Bind` seeds state from the resolved config rather than from a read, so the + destination fields should be present and the first plan after a bind should settle. Read, not run. - **Partial failure.** One member failing does not stop the others, and the error carries what was updated, deleted and failed. Reproducing it needs per-member failure injection the fake does not have. From 13f9ffbc341fae81021dc88f7bc3823e046bbc9d Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 21:24:07 +0000 Subject: [PATCH 08/47] [FS-2621] Add a cloud-capable batch case, and normalize what only a cloud run prints The batch shape had no case that could run against a workspace, which left the two claims a user is most likely to care about untested there: that repointing a destination is a recreate rather than a silent no-op, and that an update mixing an acceptable change with an unacceptable one is refused whole. Both now run live. The destination repoint plans recreate on the two immutable destination fields with their persisted values intact. The mixed update -- pausing the offline member, which the endpoint accepts, together with pausing the online one, which it does not -- is refused, and a read afterwards shows neither member moved, so the endpoint validates before it writes. Pausing the offline member alone is then applied and does not cascade. Each phase is its own template rather than a string edit on the last one, so what a phase deploys is readable without replaying the ones before it. Two replacements had to be added for the cloud run: a real set id is 32 undashed hex characters where the fake's are dashed, and the workspace host is absent from a local run entirely. Without them the goldens differ between the two runs for no reason beyond formatting. Co-authored-by: Isaac --- .../batch/databricks.yml.tmpl | 46 +++++++++++++ .../materialization_sets/batch/moved.yml.tmpl | 46 +++++++++++++ .../materialization_sets/batch/out.test.toml | 2 + .../materialization_sets/batch/output.txt | 67 +++++++++++++++++++ .../batch/paused-both.yml.tmpl | 48 +++++++++++++ .../batch/paused-offline.yml.tmpl | 47 +++++++++++++ .../materialization_sets/batch/script | 52 ++++++++++++++ .../materialization_sets/batch/test.toml | 29 ++++++++ .../materialization_sets/sawtooth/test.toml | 11 +++ .../materialization_sets/streaming/test.toml | 11 +++ 10 files changed, 359 insertions(+) create mode 100644 acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/batch/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/script create mode 100644 acceptance/bundle/resources/materialization_sets/batch/test.toml diff --git a/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl new file mode 100644 index 00000000000..83aebb13548 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl @@ -0,0 +1,46 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl new file mode 100644 index 00000000000..b5eb76c26b2 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl @@ -0,0 +1,46 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: moved_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: moved_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/out.test.toml b/acceptance/bundle/resources/materialization_sets/batch/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt new file mode 100644 index 00000000000..3dcee2486e7 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -0,0 +1,67 @@ + +>>> create_table.py danielb.fs2680.src_[UNIQUE_NAME] +Created table danielb.fs2680.src_[UNIQUE_NAME] +Inserted sample data into danielb.fs2680.src_[UNIQUE_NAME] +Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) + +=== A batch feature with both destinations creates, and a redeploy plans nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created features.batchfeat +Created materialization_sets.set1 +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + +=== Repointing the destinations is a recreate, not a silent no-op. The server cannot move a +=== materialization in place, so the set is rebuilt at the new location. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)" out.plan.json +spec:immutable old=batch_[UNIQUE_NAME] new=moved_[UNIQUE_NAME] +spec:immutable old=batch_[UNIQUE_NAME]_off new=moved_[UNIQUE_NAME]_off + +=== One request pausing both members. The offline half is a change the endpoint accepts and +=== the online half is not, so the whole request is refused. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: The 'pipeline_schedule_state' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. (400 BAD_REQUEST) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: The 'pipeline_schedule_state' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. + +Files: 2 uploaded, 0 deleted + +=== The offline member paused on its own is applied, and does not cascade to the online one +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl new file mode 100644 index 00000000000..abbf6617ef8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl @@ -0,0 +1,48 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl new file mode 100644 index 00000000000..d13a099cd7b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl @@ -0,0 +1,47 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script new file mode 100644 index 00000000000..cd8162c113f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -0,0 +1,52 @@ +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the fixture a phase deploys is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_actions() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +title "A batch feature with both destinations creates, and a redeploy plans nothing" +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +title "Repointing the destinations is a recreate, not a silent no-op. The server cannot move a" +title "materialization in place, so the set is rebuilt at the new location." +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)"' out.plan.json + +title "One request pausing both members. The offline half is a change the endpoint accepts and" +title "the online half is not, so the whole request is refused." +render paused-both.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy + +title "The offline member paused on its own is applied, and does not cascade to the online one" +render paused-offline.yml.tmpl +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/batch/test.toml b/acceptance/bundle/resources/materialization_sets/batch/test.toml new file mode 100644 index 00000000000..ac04768e226 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/test.toml @@ -0,0 +1,29 @@ +# Pinned to a workspace holding a Stream, since a Stream is not a bundle resource. Override +# MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml index 45bb2a74f39..92c3ef0e316 100644 --- a/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml @@ -6,3 +6,14 @@ Ignore = [ ".databricks", "databricks.yml", ] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' diff --git a/acceptance/bundle/resources/materialization_sets/streaming/test.toml b/acceptance/bundle/resources/materialization_sets/streaming/test.toml index 45bb2a74f39..92c3ef0e316 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/test.toml +++ b/acceptance/bundle/resources/materialization_sets/streaming/test.toml @@ -6,3 +6,14 @@ Ignore = [ ".databricks", "databricks.yml", ] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' From 7a7f205520ea877e0ac887cb58ec489992db5952 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 17 Oct 2026 21:24:53 +0000 Subject: [PATCH 09/47] [FS-2621] Record the batch case and the deployment-history blocker The deployment-history leg cannot reach a workspace at all: the bundle deployments API keeps an allowlist of resource-type prefixes and neither features nor materialization_sets is on it, so a deploy under that leg fails before any feature-engineering call is made. Worth writing down next to the run instructions, because the local fake accepts the prefix and hides it. Co-authored-by: Isaac --- .../bundle/resources/materialization_sets/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 76c7e4474a5..79adfad1f64 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -15,17 +15,24 @@ a claim about the real API, and the cloud run is what checks it. | `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | | `removal_scope` | the delete cascade is scoped per feature: a feature contributing one member can have it removed alone, while removing one member of a feature contributing two is refused | | `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates — the two are told apart by whether the persisted value is recoverable | +| `batch` | a batch feature with both destinations creates and settles; repointing the destinations is a `recreate`; an update mixing an acceptable change with an unacceptable one is refused whole, and the acceptable one applies on its own | | `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing | | `streaming` | a lone streaming online member needs no schedule and no cron, and pausing it is refused | ## Running against a workspace -`sawtooth` and `streaming` already carry `Cloud = true`. Both reference an existing `Stream`, +`batch`, `sawtooth` and `streaming` already carry `Cloud = true`, and all three pass against a workspace. Both reference an existing `Stream`, because a Stream is not a bundle resource; override `MATSET_CATALOG`, `MATSET_SCHEMA`, `MATSET_ONLINE_STORE` and `MATSET_STREAM` to move them to another workspace, and set `MATSET_PROFILE` to pin the bundle to a named profile. **`DATABRICKS_CONFIG_PROFILE` does not reach the CLI subprocess**, and the harness builds its own workspace client from the ambient environment -before any script runs, so a local cloud run needs working ambient auth to the target workspace. +before any script runs, so a local cloud run needs working ambient auth to the target workspace. `batch` +also needs `TEST_DEFAULT_WAREHOUSE_ID`, which the harness only supplies off cloud. + +The deployment-history leg (`DMS=true`) cannot run against a workspace: the bundle deployments API +rejects `features.` and `materialization_sets.` as resource-type prefixes, so every deploy of either +type fails there with `operations.resource_key must have a known resource type prefix`. Select the +`DMS=` leg until that allowlist includes them. ### The other six cases From 9bc93140da1089a257c3fe513645ce18a527be10 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 04:56:27 +0000 Subject: [PATCH 10/47] [FS-2621] Record what a workspace run proved, and the destination rule it exposed Four things ran against a workspace without a case of their own: a mixed update whose halves are both legal, the same update with either half rejected, out-of-band creation, and bind/unbind. Each needs a multi-feature set, and a multi-feature set cannot be built the way the local cases build one. That is the rule the run exposed. Every feature in a set must name the same catalog, schema and table name prefix; differing only in the prefix is refused. The fake does not check it, so membership and removal_scope hand each feature its own prefix and would both be refused against a real workspace. Noted with the other fixture deltas rather than changed, since those two cases are local-only and their claims are about reconciliation, not destinations. Bind also settles the question left open earlier: the first plan after a bind reports skip, with the offline destination suppressed the same way as on a bundle-created set. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 79adfad1f64..c9c4e66a045 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -46,21 +46,51 @@ Env.DATABRICKS_LITESWAP_TARGET = "" The env var takes the unit name only — `liteswap.Apply` prepends `testenv://liteswap/`. Passing the full traffic id routes to base staging instead, where these endpoints do not exist. -Three fixture changes are needed first, because the local names are only valid against the fake: +Four fixture changes are needed first, because the local names are only valid against the fake: 1. **Catalog and schema.** Declare `catalogs` and `schemas` resources using the harness's `UNIQUE_NAME` and reference them from the feature names, so parallel runs do not collide. 2. **Online store.** `online_store_name` must name a store that exists on the target workspace. 3. **Features must exist.** Declare the `features` resources the set members reference; the fake does not check that a member's feature exists, and the real API does. +4. **One destination for the whole set.** Every feature in a set must name the same + `catalog_name`, `schema_name` and `table_name_prefix`; differing only in the prefix is rejected + with `All features must share the same materialization destination.` The fake does not check + this, so `membership` and `removal_scope` give each feature its own prefix and would both be + refused as written. The workspace also needs a declarative feature preview enabled — `validateAnyDeclarativePreviewEnabled` rejects every create otherwise, with `declarativePreviewsNotEnabled`. +## Verified against a workspace, with no case of their own + +These ran against `liteswap-1-4` through the swap but are not committed as cases, because each one +needs a multi-feature set and the shared-destination rule above means that fixture has to be built +differently from the local ones. + +- **A mixed update whose halves are both legal is applied whole.** Pausing one feature's offline + member and removing another feature's members in the same request: both land, the surviving + members keep their ids, and the paused feature's online member stays `ACTIVE`. +- **A mixed update is atomic when either half is rejected.** Two shapes were tried -- a legal pause + plus an illegal online pause, and a legal pause plus the illegal removal of one online member -- + and in both the whole request is refused and a read afterwards shows nothing moved, including the + half that would have been accepted on its own. +- **Out-of-band creation is caught by the server, not by the planner.** The plan says `create` + because the bundle has no state for the resource; the deploy then fails with `ALREADY_EXISTS` on + the feature, and a set whose feature already has materializations fails with + `RESOURCE_ALREADY_EXISTS`. A set has no name to collide on, so that second check is the only thing + standing between an unbound set and a duplicate. +- **Bind and unbind.** Binding an out-of-band feature and set makes the next plan settle to `skip` + with no drift; the offline destination is suppressed as `input_only_not_returned` exactly as on a + bundle-created set. Unbinding leaves the workspace resource in place and the next plan reports + `create` for it again. + ## Not covered yet -- **Bind and unbind.** `Bind` seeds state from the resolved config rather than from a read, so the - destination fields should be present and the first plan after a bind should settle. Read, not run. +- **A set's materialized output tables outlive it.** `bundle destroy` removes the set and its + features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case + that wants a clean schema has to drop those itself. + - **Partial failure.** One member failing does not stop the others, and the error carries what was updated, deleted and failed. Reproducing it needs per-member failure injection the fake does not have. From 837e1d81f0b0cbfb2ce50f7b835225419e4d81a5 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 05:31:25 +0000 Subject: [PATCH 11/47] [FS-2621] Resolve a set's destinations against bundle catalogs, and guard the field list Two gaps that only show up late. CaptureUCDependencies rewrites a feature's full_name so a catalog or schema declared in the same bundle becomes an explicit dependency, and a set's four destination fields got none of that: a set pointing at a bundle-declared catalog had no edge to it, so nothing ordered the catalog first. Same shape as the feature-before-set ordering that only surfaced against a real workspace. The second is that nothing kept the hand-written recreate list exhaustive. TestNoUpdateResourcesCoverAllFields stops at HasDoUpdate and this resource has one, so a member field added to the proto later would be classified as a plain update and the endpoint would accept the request and write nothing. The new coverage test walks the SDK types instead, and pins the destination fields the change override suppresses -- both verified by deleting a rule and watching them fail. Making the fake redact input-only fields had also broken TestAll's round-trip check, which claims to skip fields configured in ignore_remote_changes and could not: it compares a pattern written foo[*].bar against a walked path foo[0].bar by string equality. Every other fake echoes input-only values back, so this resource is the first to reach it. Now matched with the pattern matcher; TestAll stays green for every resource. The offline blocks in the local fixtures were empty, which the regenerated required field validation correctly rejects and the real API rejects too. Filled in, which also gives every feature in those sets one shared destination -- the other thing the API requires. Co-authored-by: Isaac --- .../materialization_sets/basic/databricks.yml | 5 +- .../materialization_sets/basic/output.txt | 6 +- .../immutable/databricks.yml | 5 +- .../materialization_sets/immutable/output.txt | 5 +- .../materialization_sets/immutable/script | 5 +- .../membership/databricks.yml | 10 +- .../membership/output.txt | 35 +++-- .../materialization_sets/membership/script | 5 +- .../removal_scope/databricks.yml | 10 +- .../removal_scope/output.txt | 10 +- .../materialization_sets/removal_scope/script | 10 +- .../schedule_state/databricks.yml | 5 +- .../schedule_state/output.txt | 12 +- .../capture_uc_dependencies.go | 24 ++- .../capture_uc_dependencies_test.go | 67 +++++++++ bundle/direct/dresources/all_test.go | 26 +++- .../materialization_set_coverage_test.go | 139 ++++++++++++++++++ 17 files changed, 333 insertions(+), 46 deletions(-) create mode 100644 bundle/direct/dresources/materialization_set_coverage_test.go diff --git a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml index 4d1839e0d9f..c347eae6cd1 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml @@ -27,7 +27,10 @@ resources: # One feature with both destinations: the offline member a delete cascades from, and the # online member it cascades to. - feature_name: main.myschema.batchfeat - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline cron_schedule_trigger: mode: MANUAL cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/basic/output.txt b/acceptance/bundle/resources/materialization_sets/basic/output.txt index 738e5d44b5f..56102a6f16a 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/output.txt +++ b/acceptance/bundle/resources/materialization_sets/basic/output.txt @@ -38,9 +38,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged }, "feature_name": "main.myschema.batchfeat", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" } }, { diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml index 51a64ca8ab2..5580af3a18a 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml @@ -6,7 +6,10 @@ resources: set1: materialized_features: - feature_name: main.myschema.featA - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline cron_schedule_trigger: mode: MANUAL cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt index 235e5658d7c..782750f4dec 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/output.txt +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -26,7 +26,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged mode: MANUAL cron_expression: "0 0 * * * ?" - feature_name: main.myschema.featB - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script index d7bfea94358..631dffa99bf 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/script +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -15,7 +15,10 @@ trace update_file.py databricks.yml ' cron_schedule_trigger: mode: MANUAL cron_expression: "0 0 * * * ?" - feature_name: main.myschema.featB - offline_store_config: {}' + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline' trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml index b30b5cd5585..4b83a8fcce6 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml @@ -6,7 +6,10 @@ resources: set1: materialized_features: - feature_name: main.myschema.featA - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline - feature_name: main.myschema.featA online_store_config: online_store_name: danielb @@ -14,4 +17,7 @@ resources: schema_name: myschema table_name_prefix: feata - feature_name: main.myschema.featB - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index 50c3420f532..e6bbe671dcf 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -17,9 +17,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "feature_name": "main.myschema.featA", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" } }, { @@ -34,9 +34,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "feature_name": "main.myschema.featB", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" } } ] @@ -77,17 +77,17 @@ Files: 1 uploaded, 0 deleted { "feature_name": "main.myschema.featA", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" } }, { "feature_name": "main.myschema.featB", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" } } ] @@ -96,7 +96,10 @@ Files: 1 uploaded, 0 deleted === Removing a whole feature's members plans as update, not recreate, and leaves the sibling >>> update_file.py databricks.yml - feature_name: main.myschema.featA - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline >>> [CLI] bundle plan -o json @@ -125,9 +128,9 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged { "feature_name": "main.myschema.featB", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" } } ] diff --git a/acceptance/bundle/resources/materialization_sets/membership/script b/acceptance/bundle/resources/materialization_sets/membership/script index a8008fe6a6e..4fb3be105ee 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/script +++ b/acceptance/bundle/resources/materialization_sets/membership/script @@ -15,7 +15,10 @@ trace print_requests.py //materialization-sets title "Removing a whole feature's members plans as update, not recreate, and leaves the sibling" trace update_file.py databricks.yml ' - feature_name: main.myschema.featA - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline ' '' trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml index a07f6574337..822b3c390c3 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml @@ -7,10 +7,16 @@ resources: materialized_features: # F1 contributes one member, so nothing cascades from it. - feature_name: main.myschema.featF1 - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline # F2 contributes both, so a delete on its offline member reaches its online one. - feature_name: main.myschema.featF2 - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline - feature_name: main.myschema.featF2 online_store_config: online_store_name: danielb diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt index c6ca5ac3e3f..47504801859 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -8,7 +8,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Removing F1's only member is allowed: the cascade is scoped per feature, and F2's pair is === untouched because the request asks nothing about it >>> update_file.py databricks.yml - feature_name: main.myschema.featF1 - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline >>> [CLI] bundle plan -o json @@ -39,7 +42,10 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Removing F2's offline member alone is refused, because that delete cascades to F2's online === member, which the request still declares >>> update_file.py databricks.yml - feature_name: main.myschema.featF2 - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline >>> musterr [CLI] bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script index e08b7779fdb..3c35e547c58 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/script +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/script @@ -3,7 +3,10 @@ trace $CLI bundle deploy title "Removing F1's only member is allowed: the cascade is scoped per feature, and F2's pair is" title "untouched because the request asks nothing about it" trace update_file.py databricks.yml ' - feature_name: main.myschema.featF1 - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline ' '' trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json @@ -13,7 +16,10 @@ trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.mat title "Removing F2's offline member alone is refused, because that delete cascades to F2's online" title "member, which the request still declares" trace update_file.py databricks.yml ' - feature_name: main.myschema.featF2 - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline ' '' trace musterr $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml index 711a7d5f016..d4064e0b26a 100644 --- a/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml @@ -6,7 +6,10 @@ resources: set1: materialized_features: - feature_name: main.myschema.featA - offline_store_config: {} + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline pipeline_schedule_state: ACTIVE - feature_name: main.myschema.featA online_store_config: diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt index a49d10120b5..2b460ec1d0b 100644 --- a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt @@ -33,9 +33,9 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged { "feature_name": "main.myschema.featA", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" }, "pipeline_schedule_state": "ACTIVE" }, @@ -63,9 +63,9 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged { "feature_name": "main.myschema.featA", "offline_store_config": { - "catalog_name": "", - "schema_name": "", - "table_name_prefix": "" + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" }, "pipeline_schedule_state": "PAUSED" }, diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index a0e8d05fbde..f2893698bcb 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -14,13 +14,15 @@ import ( type captureUCDependencies struct{} // If a user defines a UC schema in the bundle, they can refer to it in SDP pipelines, -// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, or Vector Search -// Indexes using the `${resources.schemas..name}` syntax. Using this syntax allows +// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, Vector Search +// Indexes, Features, or Materialization Sets using the `${resources.schemas..name}` +// syntax. Using this syntax allows // TF to capture the deploy time dependency this resource has on the schema and deploy changes // to the schema before deploying the dependent resource. // // Similarly, if a user defines a UC catalog in the bundle, they can refer to it in UC schemas, -// UC Volumes, Registered Models, Model Serving Endpoints, or Vector Search Indexes using the +// UC Volumes, Registered Models, Model Serving Endpoints, Vector Search Indexes, Features, or +// Materialization Sets using the // `${resources.catalogs..name}` syntax. This captures the deploy time // dependency the resource has on the catalog. // @@ -204,6 +206,22 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia catalogName, schemaName := parts[0], parts[1] f.FullName = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] } + for _, set := range b.Config.Resources.MaterializationSets { + if set == nil { + continue + } + for i := range set.MaterializedFeatures { + member := &set.MaterializedFeatures[i] + if offline := member.OfflineStoreConfig; offline != nil { + offline.SchemaName = resolveSchema(b, offline.CatalogName, offline.SchemaName) + offline.CatalogName = resolveCatalog(b, offline.CatalogName) + } + if online := member.OnlineStoreConfig; online != nil { + online.SchemaName = resolveSchema(b, online.CatalogName, online.SchemaName) + online.CatalogName = resolveCatalog(b, online.CatalogName) + } + } + } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { continue diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go index 347f0b72caa..15094079ee2 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go @@ -365,6 +365,7 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { QualityMonitors: map[string]*resources.QualityMonitor{"nil": nil, "empty": {}}, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{"nil": nil, "empty": {}}, VectorSearchIndexes: map[string]*resources.VectorSearchIndex{"nil": nil, "empty": {}}, + MaterializationSets: map[string]*resources.MaterializationSet{"nil": nil, "empty": {}}, }, }, } @@ -373,6 +374,72 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { require.Nil(t, d) } +func TestCaptureUCDependenciesMaterializationSet(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Catalogs: map[string]*resources.Catalog{ + "my_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "mycatalog"}}, + }, + Schemas: map[string]*resources.Schema{ + "my_schema": {CreateSchema: catalog.CreateSchema{CatalogName: "mycatalog", Name: "myschema"}}, + }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "set1": {MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + { + FeatureName: "mycatalog.myschema.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "mycatalog", SchemaName: "myschema", TableNamePrefix: "feat_off", + }, + }, + { + FeatureName: "mycatalog.myschema.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{ + OnlineStoreName: "store", CatalogName: "mycatalog", + SchemaName: "myschema", TableNamePrefix: "feat", + }, + }, + // Neither destination declared in the bundle, so both pass through. + { + FeatureName: "other.other.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "other", SchemaName: "other", TableNamePrefix: "feat_off", + }, + }, + // A member with no destination at all. + {FeatureName: "mycatalog.myschema.feat"}, + }, + }}, + "nil_set": nil, + }, + }, + }, + } + + d := bundle.Apply(t.Context(), b, CaptureUCDependencies()) + require.Nil(t, d) + + schemaRef := "${resources.schemas.my_schema.name}" + catalogRef := "${resources.catalogs.my_catalog.name}" + members := b.Config.Resources.MaterializationSets["set1"].MaterializedFeatures + + assert.Equal(t, catalogRef, members[0].OfflineStoreConfig.CatalogName) + assert.Equal(t, schemaRef, members[0].OfflineStoreConfig.SchemaName) + assert.Equal(t, "feat_off", members[0].OfflineStoreConfig.TableNamePrefix) + + assert.Equal(t, catalogRef, members[1].OnlineStoreConfig.CatalogName) + assert.Equal(t, schemaRef, members[1].OnlineStoreConfig.SchemaName) + assert.Equal(t, "store", members[1].OnlineStoreConfig.OnlineStoreName) + + assert.Equal(t, "other", members[2].OfflineStoreConfig.CatalogName) + assert.Equal(t, "other", members[2].OfflineStoreConfig.SchemaName) + + assert.Nil(t, members[3].OfflineStoreConfig) + assert.Nil(t, members[3].OnlineStoreConfig) + assert.Nil(t, b.Config.Resources.MaterializationSets["nil_set"]) +} + func TestSplitUCName(t *testing.T) { tests := []struct { name string diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index c94a035d981..a2d43a7ed9f 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -967,8 +967,9 @@ func TestAll(t *testing.T) { // testIgnoreFilter encapsulates the logic for filtering fields based on ignore_remote_changes config. type testIgnoreFilter struct { - ignoreFields map[string]bool - adapter *Adapter + ignoreFields map[string]bool + ignorePatterns []*structpath.PatternNode + adapter *Adapter } // newTestIgnoreFilter creates a filter from the adapter's resource configs. @@ -977,12 +978,14 @@ type testIgnoreFilter struct { // and RemapState cannot populate them from remote state. func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { ignoreFields := make(map[string]bool) + var ignorePatterns []*structpath.PatternNode for _, cfg := range []*ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { if cfg == nil { continue } for _, p := range cfg.IgnoreRemoteChanges { ignoreFields[p.Field.String()] = true + ignorePatterns = append(ignorePatterns, p.Field) } } // Auto-include fields present in StateType but absent from RemoteType. @@ -996,7 +999,7 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { } return true }) - return &testIgnoreFilter{ignoreFields: ignoreFields, adapter: adapter} + return &testIgnoreFilter{ignoreFields: ignoreFields, ignorePatterns: ignorePatterns, adapter: adapter} } // shouldIgnore returns true if the field at the given path should be ignored. @@ -1015,7 +1018,22 @@ func (f *testIgnoreFilter) shouldIgnore(path string) bool { topLevelField = parts[0] } - return f.ignoreFields[topLevelField] + if f.ignoreFields[topLevelField] { + return true + } + + // A rule inside a slice is written as a pattern (foo[*].bar) while a walked path is indexed + // (foo[0].bar), so the lookups above never match one. Compare with the pattern matcher. + parsed, err := structpath.ParsePath(path) + if err != nil { + return false + } + for _, pattern := range f.ignorePatterns { + if parsed.HasPatternPrefix(pattern) { + return true + } + } + return false } // filterChanges returns only the changes that should not be ignored. diff --git a/bundle/direct/dresources/materialization_set_coverage_test.go b/bundle/direct/dresources/materialization_set_coverage_test.go new file mode 100644 index 00000000000..bb2caed521a --- /dev/null +++ b/bundle/direct/dresources/materialization_set_coverage_test.go @@ -0,0 +1,139 @@ +package dresources + +import ( + "reflect" + "strings" + "testing" + + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// memberFieldsTheUpdateEndpointAccepts are the member fields UpdateMaterializationSet can write, +// so they need no rule to reach the server. Everything else about a member is write-once, and the +// endpoint ignores it rather than failing, which is why an unclassified field is silent. +var memberFieldsTheUpdateEndpointAccepts = map[string]bool{ + "pipeline_schedule_state": true, +} + +// setFieldsWithoutRules are the set's own fields that no rule covers: the member slice, whose +// elements are diffed by key, and the id, which the generated config already classifies. +var setFieldsWithoutRules = map[string]bool{ + "materialized_features": true, +} + +// classifiedFields collects every field pattern either lifecycle config mentions, from both the +// hand-written and generated halves, since the planner consults them in sequence. +func classifiedFields(t *testing.T) map[string]bool { + t.Helper() + + handWritten := GetResourceConfig("materialization_sets") + require.NotNil(t, handWritten) + generated := GetGeneratedResourceConfig("materialization_sets") + require.NotNil(t, generated) + + classified := map[string]bool{} + for _, cfg := range []*ResourceLifecycleConfig{handWritten, generated} { + for _, rules := range [][]FieldRule{ + cfg.RecreateOnChanges, + cfg.IgnoreRemoteChanges, + cfg.IgnoreLocalChanges, + cfg.ProvidedIDFields, + cfg.UpdatableIDFields, + cfg.NormalizeSlash, + cfg.StableOutputFields, + } { + for _, rule := range rules { + classified[rule.Field.String()] = true + } + } + for _, rule := range cfg.BackendDefaults { + classified[rule.Field.String()] = true + } + } + return classified +} + +// memberFieldPath builds the path the planner hands to OverrideChangeDesc for one member field. +func memberFieldPath(t *testing.T, memberKey, field string) *structpath.PathNode { + t.Helper() + path, err := structpath.ParsePath( + "materialized_features[member='" + memberKey + "']." + field) + require.NoError(t, err) + return path +} + +// TestMaterializationSetAllMemberFieldsAreClassified keeps the hand-written recreate list +// exhaustive. TestNoUpdateResourcesCoverAllFields stops at HasDoUpdate, and this resource has one, +// so nothing else holds the line: a member field added to the proto later would be classified as a +// plain update, and the endpoint would accept the request and write nothing. +func TestMaterializationSetAllMemberFieldsAreClassified(t *testing.T) { + classified := classifiedFields(t) + + for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + name, ok := jsonFieldName(field) + if !ok || memberFieldsTheUpdateEndpointAccepts[name] { + continue + } + assert.Truef(t, classified["materialized_features[*]."+name], + "member field %q is declared in neither materialization_sets.yml nor its generated "+ + "half; add it to recreate_on_changes if the endpoint cannot write it, or to "+ + "memberFieldsTheUpdateEndpointAccepts if it can", + name, + ) + } +} + +// TestMaterializationSetAllSetFieldsAreClassified does the same for the set's own fields. +func TestMaterializationSetAllSetFieldsAreClassified(t *testing.T) { + classified := classifiedFields(t) + + for field := range reflect.TypeFor[ml.MaterializationSet]().Fields() { + name, ok := jsonFieldName(field) + if !ok || setFieldsWithoutRules[name] { + continue + } + assert.Truef(t, classified[name], + "set field %q is declared in neither materialization_sets.yml nor its generated half", + name, + ) + } +} + +// TestMaterializationSetDestinationFieldsAreSuppressed pins the fields OverrideChangeDesc skips. +// They are input-only, so the read never returns them and the generated rule that would absorb +// the difference cannot fire; a field dropped from this list silently re-plans as a recreate. +func TestMaterializationSetDestinationFieldsAreSuppressed(t *testing.T) { + offline := &ml.MaterializedFeature{ + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "c", SchemaName: "s", TableNamePrefix: "p"}, + } + online := &ml.MaterializedFeature{ + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "store", CatalogName: "c"}, + } + + for _, field := range []string{"catalog_name", "schema_name", "table_name_prefix"} { + _, offlineKey := materializationSetMemberKey(*offline) + _, onlineKey := materializationSetMemberKey(*online) + assert.Truef(t, + isRedactedDestinationField(memberFieldPath(t, offlineKey, "offline_store_config."+field)), + "offline_store_config.%s must be suppressed", field) + assert.Truef(t, + isRedactedDestinationField(memberFieldPath(t, onlineKey, "online_store_config."+field)), + "online_store_config.%s must be suppressed", field) + } + + // online_store_name is returned by the read, so a repoint of it must stay visible. + _, onlineKey := materializationSetMemberKey(*online) + assert.False(t, + isRedactedDestinationField(memberFieldPath(t, onlineKey, "online_store_config.online_store_name"))) +} + +func jsonFieldName(field reflect.StructField) (string, bool) { + tag := field.Tag.Get("json") + if tag == "" || tag == "-" { + return "", false + } + return strings.TrimSuffix(tag, ",omitempty"), true +} From fd2e45bb2ca7fa9a0aa8c1a5b1263e4d6504d951 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 07:27:57 +0000 Subject: [PATCH 12/47] [FS-2621] Follow the materialization set create into a long-running operation The create now returns databricks.longrunning.Operation. The generated SDK wraps that in a typed operation whose Wait polls GetOperation and unmarshals the response, so DoCreate submits and then waits, and reads the created set out of the response rather than out of the submit. A failed wait names the set from the operation metadata, which carries the id precisely so a half-finished creation can still be identified. The fake records the operation instead of answering with the set, and serves the poll at the path the client builds, /api/2.0/feature-engineering/operations/{id}. It records the operation already terminal, matching a backend whose execution is still inline. basic now prints that GET, so the goldens pin that the poll happens at all rather than passing because the request filter hides it. Co-authored-by: Isaac --- .../materialization_sets/basic/output.txt | 9 ++- .../materialization_sets/basic/script | 5 +- .../direct/dresources/materialization_set.go | 26 ++++++++- libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 3 + libs/testserver/materialization_set.go | 58 ++++++++++++++++++- 6 files changed, 98 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/basic/output.txt b/acceptance/bundle/resources/materialization_sets/basic/output.txt index 56102a6f16a..f175d9fce69 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/output.txt +++ b/acceptance/bundle/resources/materialization_sets/basic/output.txt @@ -22,7 +22,8 @@ Created materialization_sets.set1 Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //materialization-sets +=== The create returns a long-running operation, and the created set is read back from it +>>> print_requests.py --keep //materialization-sets { "method": "POST", "path": "/api/2.0/feature-engineering/materialization-sets", @@ -60,6 +61,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} + === The server assigned a set id and a member id per member >>> [CLI] bundle summary -o json { diff --git a/acceptance/bundle/resources/materialization_sets/basic/script b/acceptance/bundle/resources/materialization_sets/basic/script index 19a53b49aea..609612d8527 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/script +++ b/acceptance/bundle/resources/materialization_sets/basic/script @@ -2,7 +2,10 @@ title "Summary before deploy" trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {modified_status, materialized_features: [.materialized_features[] | {feature_name, is_online}]}' trace $CLI bundle deploy -trace print_requests.py //materialization-sets + +title "The create returns a long-running operation, and the created set is read back from it" +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations title "The server assigned a set id and a member id per member" trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {id}' diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 02bd611b611..83a01fd39ee 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "fmt" "strconv" "strings" @@ -139,14 +140,35 @@ func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml }) } +// DoCreate submits the set and waits for the long-running operation the create returns. The created +// set is readable only from that operation's response, which Wait reaches by polling GetOperation. func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { - response, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ + operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ MaterializationSet: *config, }) if err != nil { return "", nil, err } - return response.MaterializationSetId, response, nil + + response, err := operation.Wait(ctx) + if err != nil { + return "", nil, fmt.Errorf("%s: %w", materializationSetCreationTarget(operation), err) + } + if response.MaterializationSet == nil { + return "", nil, fmt.Errorf("%s: creation finished without returning the set", + materializationSetCreationTarget(operation)) + } + return response.MaterializationSet.MaterializationSetId, response.MaterializationSet, nil +} + +// materializationSetCreationTarget names what a creation was building, taking the set id from the +// operation metadata when the server supplied it and falling back to the operation name. +func materializationSetCreationTarget(operation ml.CreateMaterializationSetOperationInterface) string { + metadata, err := operation.Metadata() + if err == nil && metadata != nil && metadata.MaterializationSetId != "" { + return "materialization set " + metadata.MaterializationSetId + } + return "operation " + operation.Name() } func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, config *ml.MaterializationSet, _ *PlanEntry) (*ml.MaterializationSet, error) { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 26362384bd8..a2246db4023 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -230,6 +230,7 @@ type FakeWorkspace struct { Features map[string]ml.Feature MaterializationSets map[string]ml.MaterializationSet + Operations map[string]ml.Operation ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo VectorSearchIndexes map[string]fakeVectorSearchIndex @@ -515,6 +516,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { ModelProviderServices: map[string]catalog.ModelProviderService{}, Features: map[string]ml.Feature{}, MaterializationSets: map[string]ml.MaterializationSet{}, + Operations: map[string]ml.Operation{}, Volumes: map[string]catalog.VolumeInfo{}, Dashboards: NewEventualMap[string, *fakeDashboard](eventualConsistency), PublishedDashboards: map[string]dashboards.PublishedDashboard{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index af6673abe3d..53fb2da3b38 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -855,6 +855,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/feature-engineering/materialization-sets", func(req Request) any { return req.Workspace.MaterializationSetsCreate(req) }) + server.Handle("GET", "/api/2.0/feature-engineering/operations/{operation_id}", func(req Request) any { + return req.Workspace.OperationsGet(req.Vars["operation_id"]) + }) server.Handle("GET", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { return MapGet(req.Workspace, req.Workspace.MaterializationSets, req.Vars["materialization_set_id"]) }) diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index bcfc800d158..3933d21a1ce 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -168,7 +168,63 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { } s.MaterializationSets[set.MaterializationSetId] = set - return Response{Body: set} + + operation, err := s.recordMaterializationSetCreation(set) + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + return Response{Body: operation} +} + +// recordMaterializationSetCreation stores the operation the create returns and hands back a copy. +// The work is finished by the time the response is written, so the operation is recorded already +// terminal; a caller still has to poll it, because the created set is reachable only from there. +func (s *FakeWorkspace) recordMaterializationSetCreation(set ml.MaterializationSet) (ml.Operation, error) { + featureNames := map[string]bool{} + var featureOrder []string + for _, member := range set.MaterializedFeatures { + if !featureNames[member.FeatureName] { + featureNames[member.FeatureName] = true + featureOrder = append(featureOrder, member.FeatureName) + } + } + + metadata, err := json.Marshal(ml.MaterializationSetCreationMetadata{ + FeatureFullNames: featureOrder, + MaterializationSetId: set.MaterializationSetId, + State: ml.MaterializationSetCreationMetadataStateSucceeded, + }) + if err != nil { + return ml.Operation{}, err + } + response, err := json.Marshal(ml.CreateMaterializationSetResponse{MaterializationSet: &set}) + if err != nil { + return ml.Operation{}, err + } + + operation := ml.Operation{ + Name: "operations/" + nextUUID(), + Done: true, + Metadata: metadata, + Response: response, + } + s.Operations[operation.Name] = operation + return operation, nil +} + +// OperationsGet fakes GET /api/2.0/feature-engineering/operations/{operation_id}. +func (s *FakeWorkspace) OperationsGet(operationID string) Response { + defer s.LockUnlock()() + + operation, ok := s.Operations["operations/"+operationID] + if !ok { + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Operation with name 'operations/%s' does not exist.", operationID)) + } + return Response{Body: operation} } // effectiveCronScheduleMode treats an unset mode as MANUAL, so spelling MANUAL out is no change. From 64f32e3d3e95500912b7568fceeae3889b94a681 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 08:32:47 +0000 Subject: [PATCH 13/47] [FS-2621] Take the created set straight off the operation The create's long-running operation now carries the materialization set as its response rather than a wrapper holding one, so Wait hands back the set itself. The guard moves with it: the wrapper's set field was optional, where now an operation that settles without an id is the failure worth naming, because the id is what the engine records the resource under. Dropping the wrapper also removes the reason the generated lifecycle config had to be produced from a doctored spec. Nothing references the set's type from an output-only field any more, so generate_resources.py no longer infers that the whole type is read-only, and the config comes out correct from the spec as built. Co-authored-by: Isaac --- bundle/direct/dresources/materialization_set.go | 10 ++++++---- libs/testserver/materialization_set.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 83a01fd39ee..7c494d5be08 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -141,7 +141,7 @@ func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml } // DoCreate submits the set and waits for the long-running operation the create returns. The created -// set is readable only from that operation's response, which Wait reaches by polling GetOperation. +// set arrives as that operation's response, which Wait reaches by polling GetOperation. func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ MaterializationSet: *config, @@ -150,15 +150,17 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma return "", nil, err } - response, err := operation.Wait(ctx) + set, err := operation.Wait(ctx) if err != nil { return "", nil, fmt.Errorf("%s: %w", materializationSetCreationTarget(operation), err) } - if response.MaterializationSet == nil { + // The id is what the engine records the resource under, so an operation that settles without + // one is a failure even though the wait succeeded. + if set == nil || set.MaterializationSetId == "" { return "", nil, fmt.Errorf("%s: creation finished without returning the set", materializationSetCreationTarget(operation)) } - return response.MaterializationSet.MaterializationSetId, response.MaterializationSet, nil + return set.MaterializationSetId, set, nil } // materializationSetCreationTarget names what a creation was building, taking the set id from the diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 3933d21a1ce..297b06b454b 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -200,7 +200,7 @@ func (s *FakeWorkspace) recordMaterializationSetCreation(set ml.MaterializationS if err != nil { return ml.Operation{}, err } - response, err := json.Marshal(ml.CreateMaterializationSetResponse{MaterializationSet: &set}) + response, err := json.Marshal(set) if err != nil { return ml.Operation{}, err } From ad99f172d0f687a6612cc0add4f476caf078993c Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 17:15:25 +0000 Subject: [PATCH 14/47] [FS-2621] Recreate a set when a member's pipeline schedule state changes The set update endpoint no longer has any mutable member field: the schedule state joined the cron schedule trigger on the server's immutable list, so the endpoint reconciles membership and nothing else. A declared schedule-state change therefore has to be caught at plan time, because the in-place update the planner used to produce is now refused. The update mask is unchanged. It names only the member list, the handler never reads it, and the state is nested inside the list, so nothing about the request shape can narrow what the server inspects. Dropping the coverage test's exemption for the field exposed that a backend_defaults entry was being counted as classifying it, which it does not: a backend default only absorbs a field the config leaves unset. The collector now keeps the two apart, so removing the recreate rule fails the test. Co-authored-by: Isaac --- .../materialization_sets/batch/output.txt | 52 ++++--- .../batch/paused-both.yml.tmpl | 3 +- .../batch/paused-offline.yml.tmpl | 3 +- .../materialization_sets/batch/script | 13 +- .../schedule_state/output.txt | 127 +++++++----------- .../schedule_state/script | 24 ++-- .../materialization_sets/streaming/output.txt | 16 +-- .../materialization_sets/streaming/script | 4 +- .../configs/materialization_sets.yml | 16 +-- .../direct/dresources/materialization_set.go | 5 +- .../materialization_set_coverage_test.go | 40 +++--- libs/testserver/materialization_set.go | 58 +++----- 12 files changed, 165 insertions(+), 196 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index 3dcee2486e7..faabce600c7 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -32,32 +32,25 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged spec:immutable old=batch_[UNIQUE_NAME] new=moved_[UNIQUE_NAME] spec:immutable old=batch_[UNIQUE_NAME]_off new=moved_[UNIQUE_NAME]_off -=== One request pausing both members. The offline half is a change the endpoint accepts and -=== the online half is not, so the whole request is refused. +=== The schedule state is write-once, so pausing the feature is a recreate rather than the +=== in-place update the endpoint would refuse >>> [CLI] bundle plan -o json >>> set_actions { - "resources.materialization_sets.set1": "update" + "resources.materialization_sets.set1": "recreate" } ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: The 'pipeline_schedule_state' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. (400 BAD_REQUEST) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features -HTTP Status: 400 Bad Request -API error_code: BAD_REQUEST -API message: The 'pipeline_schedule_state' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 Files: 2 uploaded, 0 deleted - -=== The offline member paused on its own is applied, and does not cascade to the online one ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... -Updated materialization_sets.set1 -Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan -o json @@ -65,3 +58,28 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged { "resources.materialization_sets.set1": "skip" } + +=== Pausing the offline member alone plans as a recreate too, and the create refuses the +=== members of one feature disagreeing on the state +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: cannot recreate resources.materialization_sets.set1: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. + +Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl index abbf6617ef8..274bc48785f 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl @@ -24,8 +24,7 @@ resources: materialization_sets: set1: materialized_features: - # The offline member a delete cascades from, and the only one a schedule state can be - # applied to. + # The offline member a delete cascades from. - feature_name: ${resources.features.batchfeat.full_name} offline_store_config: catalog_name: $CATALOG_NAME diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl index d13a099cd7b..26b239c109f 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl @@ -24,8 +24,7 @@ resources: materialization_sets: set1: materialized_features: - # The offline member a delete cascades from, and the only one a schedule state can be - # applied to. + # The offline member a delete cascades from. - feature_name: ${resources.features.batchfeat.full_name} offline_store_config: catalog_name: $CATALOG_NAME diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script index cd8162c113f..7650bbe0d1e 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/script +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -36,17 +36,20 @@ trace $CLI bundle plan -o json > out.plan.json trace set_actions trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)"' out.plan.json -title "One request pausing both members. The offline half is a change the endpoint accepts and" -title "the online half is not, so the whole request is refused." +title "The schedule state is write-once, so pausing the feature is a recreate rather than the" +title "in-place update the endpoint would refuse" render paused-both.yml.tmpl trace $CLI bundle plan -o json > out.plan.json trace set_actions -trace musterr $CLI bundle deploy +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_actions -title "The offline member paused on its own is applied, and does not cascade to the online one" +title "Pausing the offline member alone plans as a recreate too, and the create refuses the" +title "members of one feature disagreeing on the state" render paused-offline.yml.tmpl -trace $CLI bundle deploy trace $CLI bundle plan -o json > out.plan.json trace set_actions +trace musterr $CLI bundle deploy --auto-approve rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt index 2b460ec1d0b..cade39560de 100644 --- a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt @@ -5,84 +5,52 @@ Created materialization_sets.set1 Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== Pausing the offline member is an in-place update ->>> update_file.py databricks.yml pipeline_schedule_state: ACTIVE pipeline_schedule_state: PAUSED - +=== A member the config leaves unset adopts the state the server set on create, so the next +=== plan is empty rather than reporting drift the bundle cannot write >>> [CLI] bundle plan -o json >>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json { - "resources.materialization_sets.set1": "update" + "resources.materialization_sets.set1": "skip" } ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated materialization_sets.set1 -Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //materialization-sets -{ - "method": "POST", - "path": "/api/2.0/feature-engineering/materialization-sets", - "q": { - "request_id": "[UUID]" +>>> [CLI] bundle summary -o json +[ + { + "is_online": null, + "pipeline_schedule_state": "ACTIVE" }, - "body": { - "materialized_features": [ - { - "feature_name": "main.myschema.featA", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "offline" - }, - "pipeline_schedule_state": "ACTIVE" - }, - { - "feature_name": "main.myschema.featA", - "online_store_config": { - "catalog_name": "main", - "online_store_name": "danielb", - "schema_name": "myschema", - "table_name_prefix": "feata" - } - } - ] + { + "is_online": null, + "pipeline_schedule_state": null } -} +] + +=== The state is write-once, so pausing the feature is a recreate rather than the in-place +=== update the server would refuse. Every member of the feature has to ask for the same state +>>> update_file.py databricks.yml pipeline_schedule_state: ACTIVE pipeline_schedule_state: PAUSED + +>>> update_file.py databricks.yml table_name_prefix: feata table_name_prefix: feata + pipeline_schedule_state: PAUSED + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json { - "method": "PATCH", - "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", - "q": { - "update_mask": "materialized_features" - }, - "body": { - "materialization_set_id": "[UUID]", - "materialized_features": [ - { - "feature_name": "main.myschema.featA", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "offline" - }, - "pipeline_schedule_state": "PAUSED" - }, - { - "feature_name": "main.myschema.featA", - "online_store_config": { - "catalog_name": "main", - "online_store_name": "danielb", - "schema_name": "myschema", - "table_name_prefix": "feata" - } - } - ] - } + "resources.materialization_sets.set1": "recreate" } -=== The change did not cascade: the online member keeps its own state +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + >>> [CLI] bundle summary -o json [ { @@ -91,21 +59,28 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged }, { "is_online": null, - "pipeline_schedule_state": null + "pipeline_schedule_state": "PAUSED" } ] -=== Declaring the state on the online member is refused, since the write would land on it ->>> update_file.py databricks.yml table_name_prefix: feata table_name_prefix: feata - pipeline_schedule_state: PAUSED +=== Resuming one member alone plans as a recreate too, and the create endpoint refuses the +=== members of one feature disagreeing +>>> update_file.py databricks.yml pipeline_schedule_state: PAUSED + - feature_name pipeline_schedule_state: ACTIVE + - feature_name ->>> musterr [CLI] bundle deploy +>>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: The 'pipeline_schedule_state' of feature 'main.myschema.featA' cannot be changed, because the change would be written on an online materialized feature. (400 BAD_REQUEST) -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: cannot recreate resources.materialization_sets.set1: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] HTTP Status: 400 Bad Request -API error_code: BAD_REQUEST -API message: The 'pipeline_schedule_state' of feature 'main.myschema.featA' cannot be changed, because the change would be written on an online materialized feature. +API error_code: INVALID_PARAMETER_VALUE +API message: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/script b/acceptance/bundle/resources/materialization_sets/schedule_state/script index 586fe83094e..6c65ff96bbf 100644 --- a/acceptance/bundle/resources/materialization_sets/schedule_state/script +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/script @@ -1,18 +1,26 @@ trace $CLI bundle deploy -title "Pausing the offline member is an in-place update" -trace update_file.py databricks.yml 'pipeline_schedule_state: ACTIVE' 'pipeline_schedule_state: PAUSED' +title "A member the config leaves unset adopts the state the server set on create, so the next" +title "plan is empty rather than reporting drift the bundle cannot write" trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json -trace $CLI bundle deploy -trace print_requests.py //materialization-sets - -title "The change did not cascade: the online member keeps its own state" trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' -title "Declaring the state on the online member is refused, since the write would land on it" +title "The state is write-once, so pausing the feature is a recreate rather than the in-place" +title "update the server would refuse. Every member of the feature has to ask for the same state" +trace update_file.py databricks.yml 'pipeline_schedule_state: ACTIVE' 'pipeline_schedule_state: PAUSED' trace update_file.py databricks.yml ' table_name_prefix: feata' ' table_name_prefix: feata pipeline_schedule_state: PAUSED' -trace musterr $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy --auto-approve +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' + +title "Resuming one member alone plans as a recreate too, and the create endpoint refuses the" +title "members of one feature disagreeing" +trace update_file.py databricks.yml ' pipeline_schedule_state: PAUSED + - feature_name' ' pipeline_schedule_state: ACTIVE + - feature_name' +trace musterr $CLI bundle deploy --auto-approve rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/streaming/output.txt b/acceptance/bundle/resources/materialization_sets/streaming/output.txt index 4bf6a953d61..d6959ee5a0a 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/output.txt +++ b/acceptance/bundle/resources/materialization_sets/streaming/output.txt @@ -21,7 +21,8 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-stream Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged -=== Pausing a streaming member is refused: the schedule state applies to offline members only +=== The schedule state is write-once, so declaring one on a streaming member that had none +=== plans as a recreate rather than an in-place update >>> update_file.py databricks.yml streaming_mode: pipeline_schedule_state: PAUSED streaming_mode: @@ -29,16 +30,5 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json { - "resources.materialization_sets.set1": "update" + "resources.materialization_sets.set1": "recreate" } - ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: The 'pipeline_schedule_state' of feature 'danielb.fs2680.stream_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. (400 BAD_REQUEST) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features -HTTP Status: 400 Bad Request -API error_code: BAD_REQUEST -API message: The 'pipeline_schedule_state' of feature 'danielb.fs2680.stream_[UNIQUE_NAME]' cannot be changed, because the change would be written on an online materialized feature. - -Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/streaming/script b/acceptance/bundle/resources/materialization_sets/streaming/script index 80596f64035..bf167bdaf04 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/script +++ b/acceptance/bundle/resources/materialization_sets/streaming/script @@ -29,11 +29,11 @@ trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json trace $CLI bundle deploy -title "Pausing a streaming member is refused: the schedule state applies to offline members only" +title "The schedule state is write-once, so declaring one on a streaming member that had none" +title "plans as a recreate rather than an in-place update" trace update_file.py databricks.yml ' streaming_mode:' ' pipeline_schedule_state: PAUSED streaming_mode:' trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json -trace musterr $CLI bundle deploy rm -f out.requests.txt out.plan.json diff --git a/bundle/direct/dresources/configs/materialization_sets.yml b/bundle/direct/dresources/configs/materialization_sets.yml index c85eb100d63..9034678d9b7 100644 --- a/bundle/direct/dresources/configs/materialization_sets.yml +++ b/bundle/direct/dresources/configs/materialization_sets.yml @@ -1,8 +1,7 @@ -# UpdateMaterializationSet accepts only the `materialized_features` mask, and a member's own -# fields are write-once: the server has no way to alter one in place. The spec leaves them -# OPTIONAL rather than IMMUTABLE because UpdateMaterializedFeature can still change some of them -# on a member addressed directly, so materialization_sets.generated.yml will never cover them and -# these entries are permanent. +# The set update reconciles membership and nothing else, so a member's own fields are write-once. +# The spec leaves them OPTIONAL rather than IMMUTABLE because UpdateMaterializedFeature can still +# change some of them on a member addressed directly, so materialization_sets.generated.yml will +# never cover them and these entries are permanent. # # A member added or removed whole also matches these patterns, because the matcher truncates a # pattern longer than the path. ResourceMaterializationSet.OverrideChangeDesc downgrades that case @@ -18,6 +17,8 @@ recreate_on_changes: reason: immutable - field: materialized_features[*].streaming_mode reason: immutable + - field: materialized_features[*].pipeline_schedule_state + reason: immutable - field: materialized_features[*].tags reason: immutable - field: materialized_features[*].budget_policy_id @@ -26,7 +27,6 @@ recreate_on_changes: reason: immutable backend_defaults: - # The server sets a member's schedule state on create and writes it on only one node per - # feature, so a member the config leaves unset reports a value the bundle never wrote. Without - # this the undeclared case diffs as old=nil, new=nil, remote=ACTIVE and re-plans on every deploy. + # A member the config leaves unset still reports the state the server set on create, which the + # bundle never wrote, so the undeclared case diffs as old=nil, new=nil, remote=ACTIVE. - field: materialized_features[*].pipeline_schedule_state diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 7c494d5be08..07dcc6d38b5 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -112,8 +112,9 @@ func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *s // A member added to or removed from the set is a two-segment path, // materialized_features[member='||'], which an // immutable field rule under materialized_features[*] still matches. That match counts as an - // update rather than a destroy-and-recreate, because a whole-member addition or removal is what - // the update endpoint handles. + // update rather than a destroy-and-recreate, because removing a member is what the update + // endpoint reconciles. An addition reaches the same endpoint and is refused there, naming the + // feature that matched no member. if path.Len() == 2 && path.HasPatternPrefix(materializedFeaturesPattern) { change.Action = deployplan.Update return nil diff --git a/bundle/direct/dresources/materialization_set_coverage_test.go b/bundle/direct/dresources/materialization_set_coverage_test.go index bb2caed521a..48ba18caccf 100644 --- a/bundle/direct/dresources/materialization_set_coverage_test.go +++ b/bundle/direct/dresources/materialization_set_coverage_test.go @@ -11,22 +11,17 @@ import ( "github.com/stretchr/testify/require" ) -// memberFieldsTheUpdateEndpointAccepts are the member fields UpdateMaterializationSet can write, -// so they need no rule to reach the server. Everything else about a member is write-once, and the -// endpoint ignores it rather than failing, which is why an unclassified field is silent. -var memberFieldsTheUpdateEndpointAccepts = map[string]bool{ - "pipeline_schedule_state": true, -} - // setFieldsWithoutRules are the set's own fields that no rule covers: the member slice, whose // elements are diffed by key, and the id, which the generated config already classifies. var setFieldsWithoutRules = map[string]bool{ "materialized_features": true, } -// classifiedFields collects every field pattern either lifecycle config mentions, from both the -// hand-written and generated halves, since the planner consults them in sequence. -func classifiedFields(t *testing.T) map[string]bool { +// classifiedFields collects the field patterns either lifecycle config mentions, from both the +// hand-written and generated halves, since the planner consults them in sequence. Backend defaults +// are returned apart: one absorbs a field the config leaves unset and says nothing about a value +// the config does declare. +func classifiedFields(t *testing.T) (onDeclaredValue, onlyWhenUndeclared map[string]bool) { t.Helper() handWritten := GetResourceConfig("materialization_sets") @@ -34,7 +29,8 @@ func classifiedFields(t *testing.T) map[string]bool { generated := GetGeneratedResourceConfig("materialization_sets") require.NotNil(t, generated) - classified := map[string]bool{} + onDeclaredValue = map[string]bool{} + onlyWhenUndeclared = map[string]bool{} for _, cfg := range []*ResourceLifecycleConfig{handWritten, generated} { for _, rules := range [][]FieldRule{ cfg.RecreateOnChanges, @@ -46,14 +42,14 @@ func classifiedFields(t *testing.T) map[string]bool { cfg.StableOutputFields, } { for _, rule := range rules { - classified[rule.Field.String()] = true + onDeclaredValue[rule.Field.String()] = true } } for _, rule := range cfg.BackendDefaults { - classified[rule.Field.String()] = true + onlyWhenUndeclared[rule.Field.String()] = true } } - return classified + return onDeclaredValue, onlyWhenUndeclared } // memberFieldPath builds the path the planner hands to OverrideChangeDesc for one member field. @@ -70,17 +66,17 @@ func memberFieldPath(t *testing.T, memberKey, field string) *structpath.PathNode // so nothing else holds the line: a member field added to the proto later would be classified as a // plain update, and the endpoint would accept the request and write nothing. func TestMaterializationSetAllMemberFieldsAreClassified(t *testing.T) { - classified := classifiedFields(t) + onDeclaredValue, _ := classifiedFields(t) for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { name, ok := jsonFieldName(field) - if !ok || memberFieldsTheUpdateEndpointAccepts[name] { + if !ok { continue } - assert.Truef(t, classified["materialized_features[*]."+name], - "member field %q is declared in neither materialization_sets.yml nor its generated "+ - "half; add it to recreate_on_changes if the endpoint cannot write it, or to "+ - "memberFieldsTheUpdateEndpointAccepts if it can", + assert.Truef(t, onDeclaredValue["materialized_features[*]."+name], + "member field %q has no rule covering a value the config declares; the set update "+ + "reconciles membership only, so add it to recreate_on_changes. A backend_defaults "+ + "entry does not count: it fires only while the config leaves the field unset", name, ) } @@ -88,14 +84,14 @@ func TestMaterializationSetAllMemberFieldsAreClassified(t *testing.T) { // TestMaterializationSetAllSetFieldsAreClassified does the same for the set's own fields. func TestMaterializationSetAllSetFieldsAreClassified(t *testing.T) { - classified := classifiedFields(t) + onDeclaredValue, onlyWhenUndeclared := classifiedFields(t) for field := range reflect.TypeFor[ml.MaterializationSet]().Fields() { name, ok := jsonFieldName(field) if !ok || setFieldsWithoutRules[name] { continue } - assert.Truef(t, classified[name], + assert.Truef(t, onDeclaredValue[name] || onlyWhenUndeclared[name], "set field %q is declared in neither materialization_sets.yml nor its generated half", name, ) diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 297b06b454b..a4b38fc777b 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -24,6 +24,14 @@ func materializationSetError(statusCode int, errorCode, message string) Response } } +// materializedFeatureFieldCannotBeChanged refuses a field the set update cannot write. Every +// member field is one, so the only change the endpoint applies is a removal. +func materializedFeatureFieldCannotBeChanged(featureName, fieldPath string) Response { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("The '%s' of feature '%s' cannot be changed after its materialized features "+ + "are created. Recreate the materialization set instead.", fieldPath, featureName)) +} + // materializationSetMemberKey matches a requested member to a stored one the way the backend does: // by feature name, online store and whether the trigger is streaming. Feature name alone is not // unique because one feature has an offline member plus one member per online store. @@ -75,7 +83,8 @@ func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Resp } if len(states) > 1 { response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - "All materialized features must have the same pipeline schedule state.") + "All materialized features in a BatchCreateMaterializedFeaturesRequest call must "+ + "have the same pipeline schedule state.") return &response } return nil @@ -337,40 +346,19 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res "set instead.", member.FeatureName)) } if isCronScheduleChangeRequested(member.CronScheduleTrigger, stored.CronScheduleTrigger) { - return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("The 'cron_schedule_trigger' of feature '%s' cannot be changed after "+ - "its materialized features are created. Recreate the materialization set "+ - "instead.", member.FeatureName)) + return materializedFeatureFieldCannotBeChanged(member.FeatureName, + "cron_schedule_trigger") + } + // An unset state asks for no change, so the comparison is only against a stated one. + if member.PipelineScheduleState != "" && + member.PipelineScheduleState != stored.PipelineScheduleState { + return materializedFeatureFieldCannotBeChanged(member.FeatureName, + "pipeline_schedule_state") } requestedKeys[materializationSetMemberKey(member)] = true matched = append(matched, stored) } - // Apply the one mutable field. It is written on the member it is applied to and no other, and - // applying it to an online member is refused. - scheduleStateChanges := map[string]ml.MaterializedFeaturePipelineScheduleState{} - for _, member := range requested.MaterializedFeatures { - key := materializationSetMemberKey(member) - stored := storedByKey[key] - if member.PipelineScheduleState == "" || - member.PipelineScheduleState == stored.PipelineScheduleState { - continue - } - if stored.IsOnline { - return materializationSetError(http.StatusBadRequest, "BAD_REQUEST", - fmt.Sprintf("The 'pipeline_schedule_state' of feature '%s' cannot be changed, "+ - "because the change would be written on an online materialized feature.", - member.FeatureName)) - } - if member.PipelineScheduleState != ml.MaterializedFeaturePipelineScheduleStateActive && - member.PipelineScheduleState != ml.MaterializedFeaturePipelineScheduleStatePaused { - return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("Pipeline schedule state cannot be updated to '%s'. It must be ACTIVE "+ - "or PAUSED.", member.PipelineScheduleState)) - } - scheduleStateChanges[key] = member.PipelineScheduleState - } - // Delete the members the request omits. Because a delete cascades from a feature's offline // member to its online ones, a removal that would also reach a member the request still lists // cannot be expressed and is refused before anything is written. @@ -402,15 +390,7 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res } } - members := make([]ml.MaterializedFeature, 0, len(matched)) - for _, member := range matched { - if state, changed := scheduleStateChanges[materializationSetMemberKey(member)]; changed { - member.PipelineScheduleState = state - } - members = append(members, member) - } - - existing.MaterializedFeatures = members + existing.MaterializedFeatures = matched s.MaterializationSets[setID] = existing return Response{Body: existing} } From ff2a209c2f3eb65d9c7b7eb0247b604e38919a17 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 18:09:46 +0000 Subject: [PATCH 15/47] [FS-2621] Fold the removal-scope case into membership and cover bind The two membership cases pinned one rule between them: removal_scope asserted the per-feature cascade scope and the refusal, which membership already asserted from the other direction. One fixture with a feature contributing a pair and two contributing a single member covers both refusal directions, the singleton removal and the pair removal, with one deploy cycle instead of two. The fixture also gives every feature the same destination, which the real create endpoint requires and both old fixtures violated. Bind had no case at all, only a live check. Unbinding stages the same shape as a set created out of band, so one case covers both: the plan reports create while the bundle holds no state, and binding settles the first plan to skip with the destination suppressed as on a bundle-created set. Two fake messages did not match the backend. The empty-member-list refusal now uses materializationSetRequiresMembers verbatim; the unknown-operation one is marked as the single message in the file not taken from the backend, since that is resolved in the entity-store client. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 112 ++++++----- .../materialization_sets/bind/databricks.yml | 18 ++ .../materialization_sets/bind/out.test.toml | 2 + .../materialization_sets/bind/output.txt | 53 ++++++ .../materialization_sets/bind/script | 36 ++++ .../materialization_sets/bind/test.toml | 5 + .../membership/databricks.yml | 13 +- .../membership/output.txt | 178 ++++++++++++++---- .../materialization_sets/membership/script | 81 +++++++- .../removal_scope/databricks.yml | 25 --- .../removal_scope/out.test.toml | 2 - .../removal_scope/output.txt | 60 ------ .../materialization_sets/removal_scope/script | 26 --- libs/testserver/materialization_set.go | 5 +- 14 files changed, 407 insertions(+), 209 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/bind/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/bind/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/bind/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/bind/script create mode 100644 acceptance/bundle/resources/materialization_sets/bind/test.toml delete mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml delete mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml delete mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/output.txt delete mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/script diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index c9c4e66a045..179f5067dff 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -3,94 +3,116 @@ These run locally against `libs/testserver/materialization_set.go`, which mirrors `UpdateMaterializationSet`'s reconcile semantics rather than stubbing them. That is deliberate: one `output.txt` serves both the local run and a `Cloud = true` run, so every rejection message here is -a claim about the real API, and the cloud run is what checks it. +a claim about the real API, and the cloud run is what checks it. One message is not: the unknown +operation in `OperationsGet`, which the backend resolves inside the entity-store client rather than +from `FeatureStoreMessages`. It is marked as such in the fake and no case reaches it. + +## What the update endpoint can do + +The set update reconciles membership and nothing else -- `MutableMaterializedFeatureField.All` is +empty, so every member field is write-once. That shapes most of the cases below: a declared change +to any member field is caught by the planner as a `recreate`, and the only change that reaches the +server is a removal. ## What each case pins | Case | Claim | |---|---| -| `basic` | create, then a redeploy that plans nothing — the server-assigned schedule state and the output-only member fields must not read as drift | -| `membership` | removing a whole feature's members plans as `update`, not `recreate`; removing one member of a feature on its own is refused because the delete cascades | -| `schedule_state` | pausing the offline member is an in-place update and does not cascade; declaring the state on an online member is refused | +| `basic` | create through the long-running operation, then a redeploy that plans nothing -- the server-assigned schedule state and the output-only member fields must not read as drift | +| `bind` | unbinding leaves the set in the workspace and the plan reports `create`, which is the out-of-band shape; binding it back settles the first plan to `skip`, with the destination suppressed exactly as on a bundle-created set | +| `membership` | the delete cascade is per feature and runs both ways -- removing either member of a feature's pair on its own is refused, a feature contributing one member can have it removed alone, and removing a whole pair plans as `update` with the surviving members keeping their ids | +| `schedule_state` | a member the config leaves unset adopts the state the server assigned; a declared change to it is a `recreate`, and that recreate still has to satisfy the create endpoint's rule that a feature's members agree | | `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | -| `removal_scope` | the delete cascade is scoped per feature: a feature contributing one member can have it removed alone, while removing one member of a feature contributing two is refused | -| `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates — the two are told apart by whether the persisted value is recoverable | -| `batch` | a batch feature with both destinations creates and settles; repointing the destinations is a `recreate`; an update mixing an acceptable change with an unacceptable one is refused whole, and the acceptable one applies on its own | +| `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | +| `batch` | a batch feature with both destinations creates and settles; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not | | `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing | -| `streaming` | a lone streaming online member needs no schedule and no cron, and pausing it is refused | +| `streaming` | a lone streaming online member needs no schedule and no cron; declaring a schedule state on it plans as a `recreate` rather than an in-place update | ## Running against a workspace -`batch`, `sawtooth` and `streaming` already carry `Cloud = true`, and all three pass against a workspace. Both reference an existing `Stream`, +`batch`, `sawtooth` and `streaming` carry `Cloud = true`. All three reference an existing `Stream`, because a Stream is not a bundle resource; override `MATSET_CATALOG`, `MATSET_SCHEMA`, `MATSET_ONLINE_STORE` and `MATSET_STREAM` to move them to another workspace, and set `MATSET_PROFILE` to pin the bundle to a named profile. **`DATABRICKS_CONFIG_PROFILE` does not reach the CLI subprocess**, and the harness builds its own workspace client from the ambient environment -before any script runs, so a local cloud run needs working ambient auth to the target workspace. `batch` -also needs `TEST_DEFAULT_WAREHOUSE_ID`, which the harness only supplies off cloud. +before any script runs, so a local cloud run needs working ambient auth to the target workspace. +`batch` also needs `TEST_DEFAULT_WAREHOUSE_ID`, which the harness only supplies off cloud. -The deployment-history leg (`DMS=true`) cannot run against a workspace: the bundle deployments API -rejects `features.` and `materialization_sets.` as resource-type prefixes, so every deploy of either -type fails there with `operations.resource_key must have a known resource type prefix`. Select the -`DMS=` leg until that allowlist includes them. +The deployment-history leg (`DMS=true`) cannot run **against a workspace**: the bundle deployments +API rejects `features.` and `materialization_sets.` as resource-type prefixes, so every deploy of +either type fails there with `operations.resource_key must have a known resource type prefix`. +Select the `DMS=` leg for a cloud run until that allowlist includes them. The local `DMS=true` leg +does run, because the fake only requires the key to carry a prefix at all. -### The other six cases +### The other cases -Add to `test.toml`: +`basic`, `bind`, `destination`, `immutable`, `membership` and `schedule_state` are local only. Two of +them are planner assertions a workspace cannot strengthen -- `bind` and `destination` both pin a skip +reason the planner derives with no server involvement -- so converting those buys little. The other +four do make claims about the server. + +To convert one, add to `test.toml`: ```toml Cloud = true Env.DATABRICKS_LITESWAP_TARGET = "" ``` -The env var takes the unit name only — `liteswap.Apply` prepends `testenv://liteswap/`. Passing the +The env var takes the unit name only -- `liteswap.Apply` prepends `testenv://liteswap/`. Passing the full traffic id routes to base staging instead, where these endpoints do not exist. -Four fixture changes are needed first, because the local names are only valid against the fake: +Three fixture changes are needed first, because the local names are only valid against the fake: 1. **Catalog and schema.** Declare `catalogs` and `schemas` resources using the harness's `UNIQUE_NAME` and reference them from the feature names, so parallel runs do not collide. 2. **Online store.** `online_store_name` must name a store that exists on the target workspace. -3. **Features must exist.** Declare the `features` resources the set members reference; the fake - does not check that a member's feature exists, and the real API does. -4. **One destination for the whole set.** Every feature in a set must name the same - `catalog_name`, `schema_name` and `table_name_prefix`; differing only in the prefix is rejected - with `All features must share the same materialization destination.` The fake does not check - this, so `membership` and `removal_scope` give each feature its own prefix and would both be - refused as written. - -The workspace also needs a declarative feature preview enabled — `validateAnyDeclarativePreviewEnabled` +3. **Features must exist.** Declare the `features` resources the set members reference, and + reference them as `${resources.features..full_name}` so the engine orders the feature before + the set. The fake does not check that a member's feature exists and the real API does; a bare name + also leaves the create racing the feature, which is what made `sawtooth` flaky. + +A fourth rule used to bite and no longer does: every feature in a set must name the same +`catalog_name`, `schema_name` and `table_name_prefix`, or the create is refused with +`All features must share the same materialization destination.` The fake does not check this, so the +multi-feature fixtures declare one shared destination already. + +The workspace also needs a declarative feature preview enabled -- `validateAnyDeclarativePreviewEnabled` rejects every create otherwise, with `declarativePreviewsNotEnabled`. ## Verified against a workspace, with no case of their own -These ran against `liteswap-1-4` through the swap but are not committed as cases, because each one -needs a multi-feature set and the shared-destination rule above means that fixture has to be built -differently from the local ones. - -- **A mixed update whose halves are both legal is applied whole.** Pausing one feature's offline - member and removing another feature's members in the same request: both land, the surviving - members keep their ids, and the paused feature's online member stays `ACTIVE`. -- **A mixed update is atomic when either half is rejected.** Two shapes were tried -- a legal pause - plus an illegal online pause, and a legal pause plus the illegal removal of one online member -- - and in both the whole request is refused and a read afterwards shows nothing moved, including the - half that would have been accepted on its own. +Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once +schedule state. + +- **Creation across member kinds.** A batch feature with both destinations; two features sharing one + destination with a pair each; a sawtooth triple; a lone streaming RTM member. The sad paths too: + the sawtooth pair alone, members of one feature disagreeing on schedule state, two features with + different destinations, MBM on a non-Kinesis source, and an empty body. - **Out-of-band creation is caught by the server, not by the planner.** The plan says `create` because the bundle has no state for the resource; the deploy then fails with `ALREADY_EXISTS` on the feature, and a set whose feature already has materializations fails with `RESOURCE_ALREADY_EXISTS`. A set has no name to collide on, so that second check is the only thing - standing between an unbound set and a duplicate. -- **Bind and unbind.** Binding an out-of-band feature and set makes the next plan settle to `skip` - with no drift; the offline destination is suppressed as `input_only_not_returned` exactly as on a - bundle-created set. Unbinding leaves the workspace resource in place and the next plan reports - `create` for it again. + standing between an unbound set and a duplicate. The `bind` case pins the planner half of this + locally; the server half needs a workspace. +- **Get redacts the offline destination.** All four members of a two-feature set read back with + `offline_store_config` absent, which is what the `destination` suppression exists to absorb. + +Two behaviours previously recorded here are gone, not untested: a mixed update whose halves are both +legal, and the atomicity of a mixed update when one half is rejected. Both depended on a pause being +a legal update. With no mutable member field left, the only update is a removal, so there is no +mixed update to construct. ## Not covered yet +- **A recreate carrying `PAUSED` on every member.** The two server rules it rests on were checked + live -- agreeing members create, disagreeing ones are refused -- but not `PAUSED` on create itself. + This is the one claim in `batch`'s golden a workspace has not confirmed. + - **A set's materialized output tables outlive it.** `bundle destroy` removes the set and its features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case that wants a clean schema has to drop those itself. - **Partial failure.** One member failing does not stop the others, and the error carries what was updated, deleted and failed. Reproducing it needs per-member failure injection the fake does not - have. + have. This matters more than it used to: with field updates gone, a partway failure can only be a + partway *deletion*, and nothing already deleted is put back. diff --git a/acceptance/bundle/resources/materialization_sets/bind/databricks.yml b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml new file mode 100644 index 00000000000..3d4edeacb94 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.bindfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.bindfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared diff --git a/acceptance/bundle/resources/materialization_sets/bind/out.test.toml b/acceptance/bundle/resources/materialization_sets/bind/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/bind/output.txt b/acceptance/bundle/resources/materialization_sets/bind/output.txt new file mode 100644 index 00000000000..807be1954f4 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Unbinding leaves the set in the workspace and drops it from the bundle's state, which is +=== the same shape as a set created out of band: the plan reports create, because the bundle +=== has nothing recorded for it +>>> [CLI] bundle deployment unbind set1 + +>>> [CLI] bundle plan -o json + +>>> set_plan +{ + "resources.materialization_sets.set1": "create" +} + +=== Binding the existing set takes ownership of it again +>>> [CLI] bundle deployment bind set1 [UUID] --auto-approve +Successfully bound materialization_set with an id '[UUID]' +Run 'bundle deploy' to deploy changes to your workspace + +=== The first plan after a bind settles. Bind seeds state from the resolved config rather +=== than from a read, so the destination the read never returns is suppressed here exactly as +=== it is on a set the bundle created +>>> [CLI] bundle plan -o json + +>>> set_plan +{ + "resources.materialization_sets.set1": "skip" +} + +>>> destination_changes +materialized_features[member='main.myschema.bindfeat|danielb|false'].online_store_config.catalog_name skip input_only_not_returned +materialized_features[member='main.myschema.bindfeat|danielb|false'].online_store_config.schema_name skip input_only_not_returned +materialized_features[member='main.myschema.bindfeat|danielb|false'].online_store_config.table_name_prefix skip input_only_not_returned +materialized_features[member='main.myschema.bindfeat||false'].offline_store_config skip input_only_not_returned + +=== And the deploy is a no-op rather than a rebuild +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/materialization_sets/bind/script b/acceptance/bundle/resources/materialization_sets/bind/script new file mode 100644 index 00000000000..fa69819fc52 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/script @@ -0,0 +1,36 @@ +set_plan() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +destination_changes() { + jq -r '.plan["resources.materialization_sets.set1"].changes // {} | to_entries + | map(select(.key | test("store_config"))) + | map("\(.key) \(.value.action) \(.value.reason)") | .[]' out.plan.json +} + +trace $CLI bundle deploy +set_id=$($CLI bundle summary -o json | jq -r '.resources.materialization_sets.set1.id') + +title "Unbinding leaves the set in the workspace and drops it from the bundle's state, which is" +title "the same shape as a set created out of band: the plan reports create, because the bundle" +title "has nothing recorded for it" +trace $CLI bundle deployment unbind set1 +trace $CLI bundle plan -o json > out.plan.json +trace set_plan + +title "Binding the existing set takes ownership of it again" +trace $CLI bundle deployment bind set1 "$set_id" --auto-approve + +title "The first plan after a bind settles. Bind seeds state from the resolved config rather" +title "than from a read, so the destination the read never returns is suppressed here exactly as" +title "it is on a set the bundle created" +trace $CLI bundle plan -o json > out.plan.json +trace set_plan +trace destination_changes + +title "And the deploy is a no-op rather than a rebuild" +trace $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/bind/test.toml b/acceptance/bundle/resources/materialization_sets/bind/test.toml new file mode 100644 index 00000000000..83e04113992 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/test.toml @@ -0,0 +1,5 @@ +# bind and unbind refuse to run while deployment history recording is on. +EnvMatrix.DMS = [""] + +# The case asserts plan actions and skip reasons, not request shapes. +RecordRequests = false diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml index 4b83a8fcce6..dcc1f4f2569 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml @@ -5,19 +5,26 @@ resources: materialization_sets: set1: materialized_features: + # featA contributes a pair: a delete reaching either member cascades to the other. - feature_name: main.myschema.featA offline_store_config: catalog_name: main schema_name: myschema - table_name_prefix: offline + table_name_prefix: shared - feature_name: main.myschema.featA online_store_config: online_store_name: danielb catalog_name: main schema_name: myschema - table_name_prefix: feata + table_name_prefix: shared + # featB and featC each contribute one member, so each can be removed on its own. - feature_name: main.myschema.featB offline_store_config: catalog_name: main schema_name: myschema - table_name_prefix: offline + table_name_prefix: shared + - feature_name: main.myschema.featC + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index e6bbe671dcf..429a9e93ada 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -19,7 +19,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "offline_store_config": { "catalog_name": "main", "schema_name": "myschema", - "table_name_prefix": "offline" + "table_name_prefix": "shared" } }, { @@ -28,7 +28,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "catalog_name": "main", "online_store_name": "danielb", "schema_name": "myschema", - "table_name_prefix": "feata" + "table_name_prefix": "shared" } }, { @@ -36,21 +36,29 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "offline_store_config": { "catalog_name": "main", "schema_name": "myschema", - "table_name_prefix": "offline" + "table_name_prefix": "shared" + } + }, + { + "feature_name": "main.myschema.featC", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "shared" } } ] } } -=== Removing one member of a feature that has two is refused: the delete cascades from the -=== offline member to its online one, so it would also reach a member still declared +=== The cascade is per feature and runs both ways. Removing featA's online member alone is +=== refused, because the delete also reaches the offline member the config still declares >>> update_file.py databricks.yml - feature_name: main.myschema.featA online_store_config: online_store_name: danielb catalog_name: main schema_name: myschema - table_name_prefix: feata + table_name_prefix: shared >>> musterr [CLI] bundle deploy --auto-approve @@ -64,6 +72,117 @@ API message: Changing one materialized feature of feature 'main.myschema.featA' Files: 1 uploaded, 0 deleted +=== And removing featA's offline member alone is refused for the same reason, from the other +=== direction +>>> update_file.py databricks.yml - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared + + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. + +Files: 1 uploaded, 0 deleted + +=== A feature contributing one member can have it removed alone: the cascade set is that one +=== member, and the features the request says nothing about are untouched +>>> update_file.py databricks.yml - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared + + +>>> update_file.py databricks.yml - feature_name: main.myschema.featB + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> set_members +[ + { + "feature_name": "main.myschema.featA", + "is_online": null + }, + { + "feature_name": "main.myschema.featA", + "is_online": null + }, + { + "feature_name": "main.myschema.featC", + "is_online": null + } +] + +=== Removing a whole feature's pair plans as update, not recreate, and the surviving member +=== keeps its id, so the set was reconciled rather than rebuilt +>>> update_file.py databricks.yml - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared + + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + >>> print_requests.py //materialization-sets { "method": "PATCH", @@ -79,43 +198,29 @@ Files: 1 uploaded, 0 deleted "offline_store_config": { "catalog_name": "main", "schema_name": "myschema", - "table_name_prefix": "offline" + "table_name_prefix": "shared" } }, { - "feature_name": "main.myschema.featB", + "feature_name": "main.myschema.featA", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "shared" + } + }, + { + "feature_name": "main.myschema.featC", "offline_store_config": { "catalog_name": "main", "schema_name": "myschema", - "table_name_prefix": "offline" + "table_name_prefix": "shared" } } ] } } - -=== Removing a whole feature's members plans as update, not recreate, and leaves the sibling ->>> update_file.py databricks.yml - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - - ->>> [CLI] bundle plan -o json - ->>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json -{ - "resources.materialization_sets.set1": "update" -} - ->>> [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated materialization_sets.set1 -Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //materialization-sets { "method": "PATCH", "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", @@ -126,22 +231,21 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "materialization_set_id": "[UUID]", "materialized_features": [ { - "feature_name": "main.myschema.featB", + "feature_name": "main.myschema.featC", "offline_store_config": { "catalog_name": "main", "schema_name": "myschema", - "table_name_prefix": "offline" + "table_name_prefix": "shared" } } ] } } -=== The surviving member kept its id, so the set was reconciled rather than rebuilt ->>> [CLI] bundle summary -o json +>>> set_members [ { - "feature_name": "main.myschema.featB", + "feature_name": "main.myschema.featC", "is_online": null } ] diff --git a/acceptance/bundle/resources/materialization_sets/membership/script b/acceptance/bundle/resources/materialization_sets/membership/script index 4fb3be105ee..2e2c80999b6 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/script +++ b/acceptance/bundle/resources/materialization_sets/membership/script @@ -1,32 +1,93 @@ +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + trace $CLI bundle deploy trace print_requests.py //materialization-sets -title "Removing one member of a feature that has two is refused: the delete cascades from the" -title "offline member to its online one, so it would also reach a member still declared" +title "The cascade is per feature and runs both ways. Removing featA's online member alone is" +title "refused, because the delete also reaches the offline member the config still declares" trace update_file.py databricks.yml ' - feature_name: main.myschema.featA online_store_config: online_store_name: danielb catalog_name: main schema_name: myschema - table_name_prefix: feata + table_name_prefix: shared ' '' trace musterr $CLI bundle deploy --auto-approve -trace print_requests.py //materialization-sets +rm -f "$OUT_REQUESTS" -title "Removing a whole feature's members plans as update, not recreate, and leaves the sibling" +title "And removing featA's offline member alone is refused for the same reason, from the other" +title "direction" trace update_file.py databricks.yml ' - feature_name: main.myschema.featA offline_store_config: catalog_name: main schema_name: myschema - table_name_prefix: offline + table_name_prefix: shared +' ' - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared +' +trace musterr $CLI bundle deploy --auto-approve +rm -f "$OUT_REQUESTS" + +title "A feature contributing one member can have it removed alone: the cascade set is that one" +title "member, and the features the request says nothing about are untouched" +trace update_file.py databricks.yml ' - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared +' ' - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared +' +trace update_file.py databricks.yml ' - feature_name: main.myschema.featB + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared ' '' trace $CLI bundle plan -o json > out.plan.json -trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace set_action trace $CLI bundle deploy --auto-approve -trace print_requests.py //materialization-sets +trace set_members -title "The surviving member kept its id, so the set was reconciled rather than rebuilt" -trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +title "Removing a whole feature's pair plans as update, not recreate, and the surviving member" +title "keeps its id, so the set was reconciled rather than rebuilt" +trace update_file.py databricks.yml ' - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared +' '' +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace print_requests.py //materialization-sets +trace set_members trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml deleted file mode 100644 index 822b3c390c3..00000000000 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml +++ /dev/null @@ -1,25 +0,0 @@ -bundle: - name: test-bundle - -resources: - materialization_sets: - set1: - materialized_features: - # F1 contributes one member, so nothing cascades from it. - - feature_name: main.myschema.featF1 - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - # F2 contributes both, so a delete on its offline member reaches its online one. - - feature_name: main.myschema.featF2 - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - - feature_name: main.myschema.featF2 - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: featf2 diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt deleted file mode 100644 index 47504801859..00000000000 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt +++ /dev/null @@ -1,60 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created materialization_sets.set1 -Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - -=== Removing F1's only member is allowed: the cascade is scoped per feature, and F2's pair is -=== untouched because the request asks nothing about it ->>> update_file.py databricks.yml - feature_name: main.myschema.featF1 - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - - ->>> [CLI] bundle plan -o json - ->>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json -{ - "resources.materialization_sets.set1": "update" -} - ->>> [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated materialization_sets.set1 -Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle summary -o json -[ - { - "feature_name": "main.myschema.featF2", - "is_online": null - }, - { - "feature_name": "main.myschema.featF2", - "is_online": null - } -] - -=== Removing F2's offline member alone is refused, because that delete cascades to F2's online -=== member, which the request still declares ->>> update_file.py databricks.yml - feature_name: main.myschema.featF2 - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - - ->>> musterr [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.myschema.featF2' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Changing one materialized feature of feature 'main.myschema.featF2' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. - -Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script deleted file mode 100644 index 3c35e547c58..00000000000 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/script +++ /dev/null @@ -1,26 +0,0 @@ -trace $CLI bundle deploy - -title "Removing F1's only member is allowed: the cascade is scoped per feature, and F2's pair is" -title "untouched because the request asks nothing about it" -trace update_file.py databricks.yml ' - feature_name: main.myschema.featF1 - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline -' '' -trace $CLI bundle plan -o json > out.plan.json -trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json -trace $CLI bundle deploy --auto-approve -trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' - -title "Removing F2's offline member alone is refused, because that delete cascades to F2's online" -title "member, which the request still declares" -trace update_file.py databricks.yml ' - feature_name: main.myschema.featF2 - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline -' '' -trace musterr $CLI bundle deploy --auto-approve - -rm -f out.requests.txt out.plan.json diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index a4b38fc777b..8ab412021c2 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -162,7 +162,7 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { if len(set.MaterializedFeatures) == 0 { return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - "materialized_features must not be empty") + "A materialization set requires at least one materialized feature.") } if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { return *bad @@ -230,6 +230,9 @@ func (s *FakeWorkspace) OperationsGet(operationID string) Response { operation, ok := s.Operations["operations/"+operationID] if !ok { + // Unlike every other message here, this wording is not taken from the backend: an unknown + // operation is resolved inside the entity-store client rather than from + // FeatureStoreMessages. No case reaches this branch, so nothing pins it either way. return materializationSetError(http.StatusNotFound, "NOT_FOUND", fmt.Sprintf("Operation with name 'operations/%s' does not exist.", operationID)) } From 7c64278336c136679a8d3757b9c7dd1b9b522fb1 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Mon, 19 Oct 2026 22:14:22 +0000 Subject: [PATCH 16/47] [FS-2621] Run the membership and immutable cases against a workspace Three cases now carry Cloud = true alongside batch, sawtooth and streaming. immutable and membership each held the only copy of a server refusal message no workspace had ever seen -- the member-addition refusal and the per-feature delete cascade -- and both are now confirmed against the real API, as is batch's recreate carrying PAUSED on every member, which was the last claim in the goldens a workspace had not checked. Converting them surfaced two create-path rules the fake did not model, so fixtures that the real server rejects were passing locally. Every feature in a set must ask for an identical (offline, online) destination pair, which a set mixing a feature materialized to both stores with one materialized offline only cannot satisfy; and a feature's offline and online destinations must differ in catalog, schema or prefix. Both are now mirrored in the fake, which immediately caught two fixtures in this suite. The first rule also reverses an earlier merge. membership and removal_scope were folded together on the grounds that they pinned one rule between them, but the pair rule makes their two shapes mutually exclusive within a single set, so the cascade refusal and the single-member removal cannot be one case. They are split again, each with a fixture the server accepts. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 79 +++--- .../materialization_sets/bind/databricks.yml | 2 +- .../immutable/added-member.yml.tmpl | 59 +++++ .../immutable/databricks.yml | 15 -- .../immutable/databricks.yml.tmpl | 53 ++++ .../immutable/out.test.toml | 2 +- .../materialization_sets/immutable/output.txt | 40 ++- .../materialization_sets/immutable/script | 45 ++-- .../materialization_sets/immutable/test.toml | 33 +++ .../membership/databricks.yml | 30 --- .../membership/databricks.yml.tmpl | 78 ++++++ .../membership/no-feata.yml.tmpl | 61 +++++ .../membership/no-offline.yml.tmpl | 70 +++++ .../membership/no-online.yml.tmpl | 69 +++++ .../membership/out.test.toml | 2 +- .../membership/output.txt | 241 ++---------------- .../materialization_sets/membership/script | 108 +++----- .../materialization_sets/membership/test.toml | 33 +++ .../removal_scope/databricks.yml.tmpl | 60 +++++ .../removal_scope/no-f1.yml.tmpl | 52 ++++ .../removal_scope/out.test.toml | 2 + .../removal_scope/output.txt | 36 +++ .../materialization_sets/removal_scope/script | 42 +++ .../removal_scope/test.toml | 33 +++ libs/testserver/materialization_set.go | 71 ++++++ 25 files changed, 906 insertions(+), 410 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl delete mode 100644 acceptance/bundle/resources/materialization_sets/immutable/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/immutable/test.toml delete mode 100644 acceptance/bundle/resources/materialization_sets/membership/databricks.yml create mode 100644 acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/membership/test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/script create mode 100644 acceptance/bundle/resources/materialization_sets/removal_scope/test.toml diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 179f5067dff..4e7d6e06b08 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -20,7 +20,8 @@ server is a removal. |---|---| | `basic` | create through the long-running operation, then a redeploy that plans nothing -- the server-assigned schedule state and the output-only member fields must not read as drift | | `bind` | unbinding leaves the set in the workspace and the plan reports `create`, which is the out-of-band shape; binding it back settles the first plan to `skip`, with the destination suppressed exactly as on a bundle-created set | -| `membership` | the delete cascade is per feature and runs both ways -- removing either member of a feature's pair on its own is refused, a feature contributing one member can have it removed alone, and removing a whole pair plans as `update` with the surviving members keeping their ids | +| `membership` | the delete cascade runs both ways -- removing either member of a feature's pair on its own is refused, while removing a whole pair plans as `update` and the other feature's pair keeps its ids | +| `removal_scope` | the cascade is scoped per feature: where every feature is materialized offline only, each contributes one member and can have it removed alone | | `schedule_state` | a member the config leaves unset adopts the state the server assigned; a declared change to it is a `recreate`, and that recreate still has to satisfy the create endpoint's rule that a feature's members agree | | `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | | `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | @@ -44,45 +45,65 @@ either type fails there with `operations.resource_key must have a known resource Select the `DMS=` leg for a cloud run until that allowlist includes them. The local `DMS=true` leg does run, because the fake only requires the key to carry a prefix at all. -### The other cases +### Running a cloud leg -`basic`, `bind`, `destination`, `immutable`, `membership` and `schedule_state` are local only. Two of -them are planner assertions a workspace cannot strengthen -- `bind` and `destination` both pin a skip -reason the planner derives with no server involvement -- so converting those buys little. The other -four do make claims about the server. +Six cases carry `Cloud = true`: `batch`, `sawtooth`, `streaming`, `immutable`, `membership` and +`removal_scope`. `basic`, `bind`, `destination` and `schedule_state` stay local, because every +server-dependent claim they make is already exercised by `batch`. -To convert one, add to `test.toml`: +The cases default to the shared acceptance schema; override `MATSET_CATALOG`, `MATSET_SCHEMA`, +`MATSET_ONLINE_STORE` to move them. These reach the script because the harness forwards ambient +environment to the CLI subprocess, minus the SDK's own auth variables, which it replaces with the +ones it resolved. -```toml -Cloud = true -Env.DATABRICKS_LITESWAP_TARGET = "" +```bash +TOK=$(databricks auth token -p -o json | jq -r .access_token) +DATABRICKS_CONFIG_FILE=/dev/null DATABRICKS_HOST= DATABRICKS_TOKEN="$TOK" CLOUD_ENV=aws TEST_DEFAULT_WAREHOUSE_ID= DATABRICKS_LITESWAP_TARGET= go test . -run 'TestAccept/bundle/resources/materialization_sets//DMS=$' ``` -The env var takes the unit name only -- `liteswap.Apply` prepends `testenv://liteswap/`. Passing the -full traffic id routes to base staging instead, where these endpoints do not exist. +Four parts of that are load-bearing: + +- **`DATABRICKS_CONFIG_FILE=/dev/null`.** With `RecordRequests = true` the harness proxies the run + and hands the subprocess a config carrying the *resolved config-file path*. The subprocess then + re-reads it and can fall back to `[__settings__] default_profile`, whose credentials have nothing + to do with the workspace under test. The symptom is a token refresh against + `/oidc/accounts//v1/token`. +- **A token rather than a profile.** The harness flattens its resolved auth into environment + variables, which overrides a `workspace: profile:` pinned in the bundle. A profile whose + `auth_type` is `databricks-cli` therefore gets re-resolved in the subprocess without the profile. +- **`/DMS=$`.** The `DMS=true` leg cannot run against a workspace: the bundle deployments API + rejects `features.` and `materialization_sets.` as resource-type prefixes. +- **The unit name only** in `DATABRICKS_LITESWAP_TARGET` -- `liteswap.Apply` prepends + `testenv://liteswap/`. Passing the full traffic id routes to base staging, where these endpoints + do not exist. + +The workspace needs a declarative feature preview enabled -- `validateAnyDeclarativePreviewEnabled` +rejects every create otherwise, with `declarativePreviewsNotEnabled`. -Three fixture changes are needed first, because the local names are only valid against the fake: +### Two destination rules the fixtures have to satisfy -1. **Catalog and schema.** Declare `catalogs` and `schemas` resources using the harness's - `UNIQUE_NAME` and reference them from the feature names, so parallel runs do not collide. -2. **Online store.** `online_store_name` must name a store that exists on the target workspace. -3. **Features must exist.** Declare the `features` resources the set members reference, and - reference them as `${resources.features..full_name}` so the engine orders the feature before - the set. The fake does not check that a member's feature exists and the real API does; a bare name - also leaves the create racing the feature, which is what made `sawtooth` flaky. +Both are enforced on create by `BatchMaterializationUtils`, and both are now mirrored in the fake, +so an invalid fixture fails locally instead of only on cloud. -A fourth rule used to bite and no longer does: every feature in a set must name the same -`catalog_name`, `schema_name` and `table_name_prefix`, or the create is refused with -`All features must share the same materialization destination.` The fake does not check this, so the -multi-feature fixtures declare one shared destination already. +1. **Every feature in a set must ask for an identical destination *pair*.** The compared value is + `(offlineStoreConfig, Option[onlineStoreConfig])` per feature, so matching catalog, schema and + prefix is not enough: a set cannot mix a feature materialized to both stores with one + materialized offline only. That is why `membership` gives every feature a pair and + `removal_scope` gives every feature an offline member alone -- the rule makes those two shapes + mutually exclusive within one set, which is why they cannot be one case. +2. **A feature's offline and online destinations must differ** in catalog, schema or prefix. The + fixtures suffix the offline prefix with `_off`. -The workspace also needs a declarative feature preview enabled -- `validateAnyDeclarativePreviewEnabled` -rejects every create otherwise, with `declarativePreviewsNotEnabled`. +A third thing the fake does not check: that a member's feature exists. Declare the `features` +resources and reference them as `${resources.features..full_name}`, which also orders the +feature before the set -- a bare name leaves the create racing the feature, which is what made +`sawtooth` flaky. ## Verified against a workspace, with no case of their own Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once -schedule state. +schedule state. Every `Cloud = true` case passed on that run, so their goldens -- including +`batch`'s recreate carrying `PAUSED` on both members -- are confirmed against the real API. - **Creation across member kinds.** A batch feature with both destinations; two features sharing one destination with a pair each; a sawtooth triple; a lone streaming RTM member. The sad paths too: @@ -104,10 +125,6 @@ mixed update to construct. ## Not covered yet -- **A recreate carrying `PAUSED` on every member.** The two server rules it rests on were checked - live -- agreeing members create, disagreeing ones are refused -- but not `PAUSED` on create itself. - This is the one claim in `batch`'s golden a workspace has not confirmed. - - **A set's materialized output tables outlive it.** `bundle destroy` removes the set and its features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case that wants a clean schema has to drop those itself. diff --git a/acceptance/bundle/resources/materialization_sets/bind/databricks.yml b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml index 3d4edeacb94..0a956c87748 100644 --- a/acceptance/bundle/resources/materialization_sets/bind/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml @@ -9,7 +9,7 @@ resources: offline_store_config: catalog_name: main schema_name: myschema - table_name_prefix: shared + table_name_prefix: shared_off - feature_name: main.myschema.bindfeat online_store_config: online_store_name: danielb diff --git a/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl new file mode 100644 index 00000000000..36ae6d6b313 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl @@ -0,0 +1,59 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml deleted file mode 100644 index 5580af3a18a..00000000000 --- a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml +++ /dev/null @@ -1,15 +0,0 @@ -bundle: - name: test-bundle - -resources: - materialization_sets: - set1: - materialized_features: - - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - cron_schedule_trigger: - mode: MANUAL - cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl new file mode 100644 index 00000000000..9be8c5add3c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl @@ -0,0 +1,53 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # Declared so the set can try to add a member for it. A member names a feature that must + # already exist, and the add is refused by the set update rather than by the feature lookup. + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml +++ b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt index 782750f4dec..fbcc91820d6 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/output.txt +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -1,50 +1,42 @@ +>>> create_table.py main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Created table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Inserted sample data into main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] is now visible (catalog_name=main) + >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb Created materialization_sets.set1 -Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === A member field the endpoint cannot alter is caught by the planner as a recreate, so the === request never reaches the server ->>> update_file.py databricks.yml 0 0 * * * ? 0 30 * * * ? - >>> [CLI] bundle plan -o json ->>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +>>> set_action { "resources.materialization_sets.set1": "recreate" } ->>> update_file.py databricks.yml 0 30 * * * ? 0 0 * * * ? - === Adding a member is routed to update, where the server refuses it by name rather than === recreating the set and destroying the members the config left alone ->>> update_file.py databricks.yml cron_schedule_trigger: - mode: MANUAL - cron_expression: "0 0 * * * ?" cron_schedule_trigger: - mode: MANUAL - cron_expression: "0 0 * * * ?" - - feature_name: main.myschema.featB - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline - >>> [CLI] bundle plan -o json ->>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +>>> set_action { "resources.materialization_sets.set1": "update" } ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features cannot be added to an existing materialization set, and feature 'main.myschema.featB' does not match any member of this set. Recreate the set instead. (400 INVALID_PARAMETER_VALUE) +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Materialized features cannot be added to an existing materialization set, and feature 'main.myschema.featB' does not match any member of this set. Recreate the set instead. +API message: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script index 631dffa99bf..08de9633a63 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/script +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -1,26 +1,41 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-materialization_set_acceptance_test}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the fixture a phase deploys is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + trace $CLI bundle deploy title "A member field the endpoint cannot alter is caught by the planner as a recreate, so the" title "request never reaches the server" -trace update_file.py databricks.yml '0 0 * * * ?' '0 30 * * * ?' +sed -i 's/0 0 \* \* \* ?/0 30 * * * ?/' databricks.yml trace $CLI bundle plan -o json > out.plan.json -trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json -trace update_file.py databricks.yml '0 30 * * * ?' '0 0 * * * ?' +trace set_action title "Adding a member is routed to update, where the server refuses it by name rather than" title "recreating the set and destroying the members the config left alone" -trace update_file.py databricks.yml ' cron_schedule_trigger: - mode: MANUAL - cron_expression: "0 0 * * * ?"' ' cron_schedule_trigger: - mode: MANUAL - cron_expression: "0 0 * * * ?" - - feature_name: main.myschema.featB - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: offline' +render added-member.yml.tmpl trace $CLI bundle plan -o json > out.plan.json -trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json -trace musterr $CLI bundle deploy +trace set_action +trace musterr $CLI bundle deploy --auto-approve rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/test.toml b/acceptance/bundle/resources/materialization_sets/immutable/test.toml new file mode 100644 index 00000000000..5ad977e46d0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/test.toml @@ -0,0 +1,33 @@ +# Pinned to the acceptance schema the materialization-set cases share. Override MATSET_CATALOG / +# MATSET_SCHEMA to point it elsewhere. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml deleted file mode 100644 index dcc1f4f2569..00000000000 --- a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml +++ /dev/null @@ -1,30 +0,0 @@ -bundle: - name: test-bundle - -resources: - materialization_sets: - set1: - materialized_features: - # featA contributes a pair: a delete reaching either member cascades to the other. - - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared - # featB and featC each contribute one member, so each can be removed on its own. - - feature_name: main.myschema.featB - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featC - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl new file mode 100644 index 00000000000..340901165da --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl @@ -0,0 +1,78 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Both features are materialized to both stores, so either member of a pair cascades to the other. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.feata.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl new file mode 100644 index 00000000000..d8d5e2f7c27 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl @@ -0,0 +1,61 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's whole pair dropped; featb's pair survives. + materialized_features: + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl new file mode 100644 index 00000000000..4bee71b4545 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl @@ -0,0 +1,70 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's offline member dropped on its own. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl new file mode 100644 index 00000000000..5895db5f4cf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl @@ -0,0 +1,69 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's online member dropped on its own. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/out.test.toml b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/out.test.toml +++ b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index 429a9e93ada..0ff01f5eb2d 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -1,131 +1,45 @@ +>>> create_table.py main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Created table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Inserted sample data into main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] is now visible (catalog_name=main) + >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb Created materialization_sets.set1 -Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //materialization-sets -{ - "method": "POST", - "path": "/api/2.0/feature-engineering/materialization-sets", - "q": { - "request_id": "[UUID]" - }, - "body": { - "materialized_features": [ - { - "feature_name": "main.myschema.featA", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - }, - { - "feature_name": "main.myschema.featA", - "online_store_config": { - "catalog_name": "main", - "online_store_name": "danielb", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - }, - { - "feature_name": "main.myschema.featB", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - }, - { - "feature_name": "main.myschema.featC", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - } - ] - } -} +Files: 5 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged -=== The cascade is per feature and runs both ways. Removing featA's online member alone is +=== The cascade is per feature and runs both ways. Removing feata's online member alone is === refused, because the delete also reaches the offline member the config still declares ->>> update_file.py databricks.yml - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - >>> musterr [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. +API message: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. Files: 1 uploaded, 0 deleted -=== And removing featA's offline member alone is refused for the same reason, from the other +=== And removing feata's offline member alone is refused for the same reason, from the other === direction ->>> update_file.py databricks.yml - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - >>> musterr [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Changing one materialized feature of feature 'main.myschema.featA' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. +API message: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. Files: 1 uploaded, 0 deleted -=== A feature contributing one member can have it removed alone: the cascade set is that one -=== member, and the features the request says nothing about are untouched ->>> update_file.py databricks.yml - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - ->>> update_file.py databricks.yml - feature_name: main.myschema.featB - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - +=== Removing a whole feature's pair plans as update, not recreate, and the other feature's +=== members keep their ids, so the set was reconciled rather than rebuilt >>> [CLI] bundle plan -o json >>> set_action @@ -134,126 +48,19 @@ Files: 1 uploaded, 0 deleted } >>> [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... Updated materialization_sets.set1 Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> set_members [ { - "feature_name": "main.myschema.featA", - "is_online": null - }, - { - "feature_name": "main.myschema.featA", + "feature_name": "${resources.features.featb.full_name}", "is_online": null }, { - "feature_name": "main.myschema.featC", + "feature_name": "${resources.features.featb.full_name}", "is_online": null } ] - -=== Removing a whole feature's pair plans as update, not recreate, and the surviving member -=== keeps its id, so the set was reconciled rather than rebuilt ->>> update_file.py databricks.yml - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - ->>> [CLI] bundle plan -o json - ->>> set_action -{ - "resources.materialization_sets.set1": "update" -} - ->>> [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated materialization_sets.set1 -Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //materialization-sets -{ - "method": "PATCH", - "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", - "q": { - "update_mask": "materialized_features" - }, - "body": { - "materialization_set_id": "[UUID]", - "materialized_features": [ - { - "feature_name": "main.myschema.featA", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - }, - { - "feature_name": "main.myschema.featA", - "online_store_config": { - "catalog_name": "main", - "online_store_name": "danielb", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - }, - { - "feature_name": "main.myschema.featC", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", - "q": { - "update_mask": "materialized_features" - }, - "body": { - "materialization_set_id": "[UUID]", - "materialized_features": [ - { - "feature_name": "main.myschema.featC", - "offline_store_config": { - "catalog_name": "main", - "schema_name": "myschema", - "table_name_prefix": "shared" - } - } - ] - } -} - ->>> set_members -[ - { - "feature_name": "main.myschema.featC", - "is_online": null - } -] - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.materialization_sets.set1 - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted diff --git a/acceptance/bundle/resources/materialization_sets/membership/script b/acceptance/bundle/resources/materialization_sets/membership/script index 2e2c80999b6..b684f53cc36 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/script +++ b/acceptance/bundle/resources/materialization_sets/membership/script @@ -1,94 +1,52 @@ -set_members() { - $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-materialization_set_acceptance_test}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" +export PREFIX="mat_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the membership a phase declares is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml } set_action() { jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json } +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + trace $CLI bundle deploy -trace print_requests.py //materialization-sets -title "The cascade is per feature and runs both ways. Removing featA's online member alone is" +title "The cascade is per feature and runs both ways. Removing feata's online member alone is" title "refused, because the delete also reaches the offline member the config still declares" -trace update_file.py databricks.yml ' - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' '' +render no-online.yml.tmpl trace musterr $CLI bundle deploy --auto-approve -rm -f "$OUT_REQUESTS" -title "And removing featA's offline member alone is refused for the same reason, from the other" +title "And removing feata's offline member alone is refused for the same reason, from the other" title "direction" -trace update_file.py databricks.yml ' - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' ' - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' +render no-offline.yml.tmpl trace musterr $CLI bundle deploy --auto-approve -rm -f "$OUT_REQUESTS" -title "A feature contributing one member can have it removed alone: the cascade set is that one" -title "member, and the features the request says nothing about are untouched" -trace update_file.py databricks.yml ' - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' ' - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' -trace update_file.py databricks.yml ' - feature_name: main.myschema.featB - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' '' +title "Removing a whole feature's pair plans as update, not recreate, and the other feature's" +title "members keep their ids, so the set was reconciled rather than rebuilt" +render no-feata.yml.tmpl trace $CLI bundle plan -o json > out.plan.json trace set_action trace $CLI bundle deploy --auto-approve trace set_members -title "Removing a whole feature's pair plans as update, not recreate, and the surviving member" -title "keeps its id, so the set was reconciled rather than rebuilt" -trace update_file.py databricks.yml ' - feature_name: main.myschema.featA - offline_store_config: - catalog_name: main - schema_name: myschema - table_name_prefix: shared - - feature_name: main.myschema.featA - online_store_config: - online_store_name: danielb - catalog_name: main - schema_name: myschema - table_name_prefix: shared -' '' -trace $CLI bundle plan -o json > out.plan.json -trace set_action -trace $CLI bundle deploy --auto-approve -trace print_requests.py //materialization-sets -trace set_members - -trace $CLI bundle destroy --auto-approve - -rm -f out.requests.txt out.plan.json +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/membership/test.toml b/acceptance/bundle/resources/materialization_sets/membership/test.toml new file mode 100644 index 00000000000..d50a94d5701 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/test.toml @@ -0,0 +1,33 @@ +# Pinned to the acceptance schema the materialization-set cases share. Override MATSET_CATALOG / +# MATSET_SCHEMA / MATSET_ONLINE_STORE to point it elsewhere. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl new file mode 100644 index 00000000000..6d2acd4fe7f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl @@ -0,0 +1,60 @@ +bundle: + name: test-matset-removal-scope-$UNIQUE_NAME + +resources: + features: + featf1: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf1_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featf2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf2_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Each feature is materialized offline only, so each contributes exactly one member. + materialized_features: + - feature_name: ${resources.features.featf1.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featf2.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl b/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl new file mode 100644 index 00000000000..ac6257da73e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl @@ -0,0 +1,52 @@ +bundle: + name: test-matset-removal-scope-$UNIQUE_NAME + +resources: + features: + featf1: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf1_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featf2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf2_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # featf1's only member dropped; featf2 is untouched. + materialized_features: + - feature_name: ${resources.features.featf2.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt new file mode 100644 index 00000000000..771a19bfec8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -0,0 +1,36 @@ + +>>> create_table.py main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Created table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Inserted sample data into main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-removal-scope-[UNIQUE_NAME]/default/files... +Created features.featf1 +Created features.featf2 +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A feature contributing one member can have it removed alone: the cascade set is that one +=== member, and the feature the request says nothing about is untouched +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-removal-scope-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> set_members +[ + { + "feature_name": "${resources.features.featf2.full_name}", + "is_online": null + } +] diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script new file mode 100644 index 00000000000..f3981736964 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/script @@ -0,0 +1,42 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-materialization_set_acceptance_test}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" +export PREFIX="mat_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the membership a phase declares is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +title "A feature contributing one member can have it removed alone: the cascade set is that one" +title "member, and the feature the request says nothing about is untouched" +render no-f1.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace set_members + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml new file mode 100644 index 00000000000..d50a94d5701 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml @@ -0,0 +1,33 @@ +# Pinned to the acceptance schema the materialization-set cases share. Override MATSET_CATALOG / +# MATSET_SCHEMA / MATSET_ONLINE_STORE to point it elsewhere. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 8ab412021c2..d6aeceefeac 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -90,6 +90,74 @@ func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Resp return nil } +// featureMaterializationDestination renders the destination pair the create compares across +// features: a feature's offline config plus the online config it is also materialized to, if any. +func featureMaterializationDestination(members []ml.MaterializedFeature) string { + offline, online := "", "" + for _, member := range members { + if cfg := member.OfflineStoreConfig; cfg != nil { + offline = cfg.CatalogName + "." + cfg.SchemaName + "." + cfg.TableNamePrefix + } + if cfg := member.OnlineStoreConfig; cfg != nil { + online = cfg.OnlineStoreName + ":" + cfg.CatalogName + "." + cfg.SchemaName + "." + + cfg.TableNamePrefix + } + } + return offline + "|" + online +} + +// validateSharedMaterializationDestination mirrors the create-path rule that every feature in a +// set asks for an identical destination pair. Because the pair carries the online config, a set +// cannot mix a feature materialized to both stores with one materialized offline only. +func validateSharedMaterializationDestination(members []ml.MaterializedFeature) *Response { + membersByFeature := map[string][]ml.MaterializedFeature{} + featureOrder := []string{} + for _, member := range members { + if _, seen := membersByFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + membersByFeature[member.FeatureName] = append(membersByFeature[member.FeatureName], member) + } + + destinations := map[string]bool{} + for _, featureName := range featureOrder { + destinations[featureMaterializationDestination(membersByFeature[featureName])] = true + } + if len(destinations) > 1 { + response := materializationSetError(http.StatusBadRequest, "BAD_REQUEST", + "All features must share the same materialization destination.") + return &response + } + return validateOfflineAndOnlineDestinationsDiffer(members) +} + +// validateOfflineAndOnlineDestinationsDiffer mirrors the create-path rule that a feature +// materialized to both stores must separate them by catalog, schema or prefix. +func validateOfflineAndOnlineDestinationsDiffer(members []ml.MaterializedFeature) *Response { + var offline *ml.OfflineStoreConfig + var online *ml.OnlineStoreConfig + for _, member := range members { + if member.OfflineStoreConfig != nil { + offline = member.OfflineStoreConfig + } + if member.OnlineStoreConfig != nil { + online = member.OnlineStoreConfig + } + } + if offline == nil || online == nil { + return nil + } + if online.CatalogName != offline.CatalogName || online.SchemaName != offline.SchemaName || + online.TableNamePrefix != offline.TableNamePrefix { + return nil + } + response := materializationSetError(http.StatusBadRequest, "BAD_REQUEST", + fmt.Sprintf("Offline and online materialization destinations must differ. Found "+ + "identical catalog names '%s', schema names '%s', and table name prefixes '%s'.", + offline.CatalogName, offline.SchemaName, offline.TableNamePrefix)) + return &response +} + // sawtoothMaterializedFeatureCount is how many materialized features a sawtooth window produces: // an offline batch one, an online batch one and an online streaming one. const sawtoothMaterializedFeatureCount = 3 @@ -167,6 +235,9 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { return *bad } + if bad := validateSharedMaterializationDestination(set.MaterializedFeatures); bad != nil { + return *bad + } if bad := s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures); bad != nil { return *bad } From d46ed36b9d6c2636dedb9695f39fd424de54defe Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Tue, 20 Oct 2026 06:22:33 +0000 Subject: [PATCH 17/47] [FS-2621] Wait on the materialization set update and delete operations Update and delete now return a long-running operation like create already did, so DoUpdate and DoDelete wait for theirs. The operation metadata type is shared across the three verbs, so the helper naming what an operation was changing is shared too. All three verbs also take an idempotency token the server reserves an operation record against, and the adapter supplies one per call. The fake's update and delete return operations to match, and the delete gets a handler of its own because the generic map delete returns an empty body. basic and removal_scope now pin the delete's and the update's operation round trip, which no golden asserted before. Co-authored-by: Isaac --- .../configs/materialization_set.yml.tmpl | 34 ++++++ .../materialization_set.yml.tmpl-cleanup.sh | 6 + .../materialization_set.yml.tmpl-init.sh | 14 +++ .../resources/materialization_sets/README.md | 32 +++++- .../materialization_sets/basic/output.txt | 14 ++- .../materialization_sets/basic/script | 4 +- .../materialization_sets/immutable/output.txt | 2 +- .../membership/output.txt | 4 +- .../removal_scope/output.txt | 34 ++++++ .../materialization_sets/removal_scope/script | 7 ++ bundle/direct/dresources/all_test.go | 5 +- .../direct/dresources/materialization_set.go | 48 ++++++-- .../dresources/materialization_set_test.go | 39 +++++++ libs/testserver/handlers.go | 2 +- libs/testserver/materialization_set.go | 105 +++++++++++------- 15 files changed, 289 insertions(+), 61 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/materialization_set.yml.tmpl create mode 100755 acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh create mode 100755 acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl new file mode 100644 index 00000000000..9966eadc384 --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + features: + foo: + full_name: main.matset_inv_$UNIQUE_NAME.test_feature + source: + delta_table_source: + full_name: main.matset_inv_$UNIQUE_NAME.source_table + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + foo: + materialized_features: + - feature_name: ${resources.features.foo.full_name} + offline_store_config: + catalog_name: main + schema_name: matset_inv_$UNIQUE_NAME + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh new file mode 100755 index 00000000000..80dcb68a26b --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# The offline materialization leaves its own tables behind, so the schema is dropped with CASCADE. +echo "Cleaning up temporary source table" +$CLI tables delete main.matset_inv_$UNIQUE_NAME.source_table || true +$CLI schemas delete main.matset_inv_$UNIQUE_NAME --force || true diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh new file mode 100755 index 00000000000..18cb15f25ae --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# The feature's delta_table_source must already exist, and the invariant harness provisions no +# tables of its own. +echo "Creating temporary source table: main.matset_inv_$UNIQUE_NAME.source_table" + +$CLI schemas create matset_inv_$UNIQUE_NAME main -o json | jq '{full_name}' + +# MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /api/... to a filesystem path. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.matset_inv_$UNIQUE_NAME.source_table (id INT, value STRING, timestamp TIMESTAMP) USING DELTA\", + \"wait_timeout\": \"45s\" + }" > /dev/null diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 4e7d6e06b08..e0e7428221b 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -14,14 +14,29 @@ empty, so every member field is write-once. That shapes most of the cases below: to any member field is caught by the planner as a `recreate`, and the only change that reaches the server is a removal. +## Create, update and delete are all long-running + +Each write verb returns a `databricks.longrunning.Operation` and the adapter waits on it, so the set +a create or update produces is read off the operation rather than the response body; a delete's +operation settles with no response at all. Execution is synchronous server-side, so the operation is +already terminal when the call returns and the wait settles on its first poll. + +Each verb also takes a `request_id`, and the server reserves an operation record against it before +acting: a repeat carrying a token already used gets the first operation back and changes nothing. +The adapter mints one per call, so the SDK's own retry of a lost response resolves to the first +operation instead of acting twice. The engine's `retryOnTransient` calls the adapter afresh and so +mints a new token; that is safe here because it never retries a create without the adapter opting +in, and re-running an update or a delete reconciles to the same state. No case replays a token, +because every adapter call mints its own. + ## What each case pins | Case | Claim | |---|---| -| `basic` | create through the long-running operation, then a redeploy that plans nothing -- the server-assigned schedule state and the output-only member fields must not read as drift | +| `basic` | create through the long-running operation, then a redeploy that plans nothing -- the server-assigned schedule state and the output-only member fields must not read as drift; the destroy's operation round trip | | `bind` | unbinding leaves the set in the workspace and the plan reports `create`, which is the out-of-band shape; binding it back settles the first plan to `skip`, with the destination suppressed exactly as on a bundle-created set | | `membership` | the delete cascade runs both ways -- removing either member of a feature's pair on its own is refused, while removing a whole pair plans as `update` and the other feature's pair keeps its ids | -| `removal_scope` | the cascade is scoped per feature: where every feature is materialized offline only, each contributes one member and can have it removed alone | +| `removal_scope` | the cascade is scoped per feature: where every feature is materialized offline only, each contributes one member and can have it removed alone; the update's operation round trip | | `schedule_state` | a member the config leaves unset adopts the state the server assigned; a declared change to it is a `recreate`, and that recreate still has to satisfy the create endpoint's rule that a feature's members agree | | `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | | `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | @@ -129,7 +144,14 @@ mixed update to construct. features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case that wants a clean schema has to drop those itself. -- **Partial failure.** One member failing does not stop the others, and the error carries what was - updated, deleted and failed. Reproducing it needs per-member failure injection the fake does not - have. This matters more than it used to: with field updates gone, a partway failure can only be a +- **Partial failure.** One member failing does not stop the others, and the operation settles + FAILED carrying what was updated, deleted and failed in its metadata's `update_progress` or + `deletion_progress`. Reproducing it needs per-member failure injection the fake does not have. + This matters more than it used to: with field updates gone, a partway failure can only be a partway *deletion*, and nothing already deleted is put back. + +- **A settled-but-failed operation.** Every refusal these cases reach is an HTTP error on the call + itself, raised before the operation record exists, so the wait is never what fails. The branch + that reports a failed operation -- naming the set from the operation metadata -- is covered only + by `TestMaterializationSetOperationTarget`, and the same injection the partial-failure case needs + would cover it end to end. diff --git a/acceptance/bundle/resources/materialization_sets/basic/output.txt b/acceptance/bundle/resources/materialization_sets/basic/output.txt index f175d9fce69..2fe3d5fb683 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/output.txt +++ b/acceptance/bundle/resources/materialization_sets/basic/output.txt @@ -89,6 +89,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged "resources.materialization_sets.set1": "skip" } +=== The delete returns a long-running operation too, which the destroy waits on >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.features.batchfeat @@ -98,8 +99,17 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted ->>> print_requests.py //materialization-sets +>>> print_requests.py --keep //materialization-sets { "method": "DELETE", - "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]" + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]" + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" } diff --git a/acceptance/bundle/resources/materialization_sets/basic/script b/acceptance/bundle/resources/materialization_sets/basic/script index 609612d8527..cd50c2da629 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/script +++ b/acceptance/bundle/resources/materialization_sets/basic/script @@ -18,7 +18,9 @@ title "The schedule state the server assigned must not read as drift" trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +title "The delete returns a long-running operation too, which the destroy waits on" trace $CLI bundle destroy --auto-approve -trace print_requests.py //materialization-sets +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt index fbcc91820d6..9d2ae096477 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/output.txt +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -34,7 +34,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. (400 INVALID_PARAMETER_VALUE) -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE API message: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index 0ff01f5eb2d..c7215b0d753 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -18,7 +18,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE API message: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. @@ -31,7 +31,7 @@ Files: 1 uploaded, 0 deleted Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?update_mask=materialized_features +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE API message: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt index 771a19bfec8..22df5d4e500 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -34,3 +34,37 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "is_online": null } ] + +=== The update returns a long-running operation, and the reconciled set is read back from it +>>> print_requests.py --keep //materialization-sets +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]", + "update_mask": "materialized_features" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.materialization_set_acceptance_test.featf2_[UNIQUE_NAME]", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "materialization_set_acceptance_test", + "table_name_prefix": "mat_[UNIQUE_NAME]" + } + } + ] + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script index f3981736964..e0bd2503606 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/script +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/script @@ -31,6 +31,9 @@ render databricks.yml.tmpl trace $CLI bundle deploy +# The create's requests are pinned by the basic case, so drop them and let the update's stand alone. +rm -f "$OUT_REQUESTS" + title "A feature contributing one member can have it removed alone: the cascade set is that one" title "member, and the feature the request says nothing about is untouched" render no-f1.yml.tmpl @@ -39,4 +42,8 @@ trace set_action trace $CLI bundle deploy --auto-approve trace set_members +title "The update returns a long-running operation, and the reconciled set is read back from it" +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + rm -f out.plan.json diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index a2d43a7ed9f..b48b1546f35 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -131,8 +131,9 @@ var testConfig map[string]any = map[string]any{ FeatureName: "main.default.my_feature", CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, OfflineStoreConfig: &ml.OfflineStoreConfig{ - CatalogName: "main", - SchemaName: "default", + CatalogName: "main", + SchemaName: "default", + TableNamePrefix: "offline", }, }, { diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 07dcc6d38b5..62b701b0205 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/google/uuid" ) // https://docs.databricks.com/api/workspace/featureengineering @@ -144,8 +145,12 @@ func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml // DoCreate submits the set and waits for the long-running operation the create returns. The created // set arrives as that operation's response, which Wait reaches by polling GetOperation. func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { + // Each verb takes an idempotency token, and a repeat carrying one already used returns the + // operation the first request created rather than acting again. Minted here so the token the + // server sees is the one this call chose. operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ MaterializationSet: *config, + RequestId: uuid.NewString(), }) if err != nil { return "", nil, err @@ -153,20 +158,27 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma set, err := operation.Wait(ctx) if err != nil { - return "", nil, fmt.Errorf("%s: %w", materializationSetCreationTarget(operation), err) + return "", nil, fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), err) } // The id is what the engine records the resource under, so an operation that settles without // one is a failure even though the wait succeeded. if set == nil || set.MaterializationSetId == "" { return "", nil, fmt.Errorf("%s: creation finished without returning the set", - materializationSetCreationTarget(operation)) + materializationSetOperationTarget(operation)) } return set.MaterializationSetId, set, nil } -// materializationSetCreationTarget names what a creation was building, taking the set id from the -// operation metadata when the server supplied it and falling back to the operation name. -func materializationSetCreationTarget(operation ml.CreateMaterializationSetOperationInterface) string { +// materializationSetOperation is what a create, update and delete operation have in common: the +// three generated interfaces share these two methods and differ only in what Wait returns. +type materializationSetOperation interface { + Name() string + Metadata() (*ml.MaterializationSetOperationMetadata, error) +} + +// materializationSetOperationTarget names the set an operation was changing, taking the set id from +// the operation metadata when the server supplied it and falling back to the operation name. +func materializationSetOperationTarget(operation materializationSetOperation) string { metadata, err := operation.Metadata() if err == nil && metadata != nil && metadata.MaterializationSetId != "" { return "materialization set " + metadata.MaterializationSetId @@ -174,19 +186,41 @@ func materializationSetCreationTarget(operation ml.CreateMaterializationSetOpera return "operation " + operation.Name() } +// DoUpdate reconciles the set's membership and waits for the long-running operation the update +// returns. The reconciled set arrives as that operation's response. func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, config *ml.MaterializationSet, _ *PlanEntry) (*ml.MaterializationSet, error) { update := *config update.MaterializationSetId = id - return r.client.FeatureEngineering.UpdateMaterializationSet(ctx, ml.UpdateMaterializationSetRequest{ + operation, err := r.client.FeatureEngineering.UpdateMaterializationSet(ctx, ml.UpdateMaterializationSetRequest{ MaterializationSetId: id, MaterializationSet: update, UpdateMask: *fieldmask.New(materializationSetUpdatableFields), + RequestId: uuid.NewString(), }) + if err != nil { + return nil, err + } + + set, err := operation.Wait(ctx) + if err != nil { + return nil, fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), err) + } + return set, nil } +// DoDelete deletes the set and waits for the long-running operation the delete returns, which +// settles with no response body. func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ *ml.MaterializationSet) error { - return r.client.FeatureEngineering.DeleteMaterializationSet(ctx, ml.DeleteMaterializationSetRequest{ + operation, err := r.client.FeatureEngineering.DeleteMaterializationSet(ctx, ml.DeleteMaterializationSetRequest{ MaterializationSetId: id, + RequestId: uuid.NewString(), }) + if err != nil { + return err + } + if err := operation.Wait(ctx); err != nil { + return fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), err) + } + return nil } diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go index 02d92cfe32b..8458ee2be14 100644 --- a/bundle/direct/dresources/materialization_set_test.go +++ b/bundle/direct/dresources/materialization_set_test.go @@ -1,6 +1,7 @@ package dresources import ( + "errors" "testing" "github.com/databricks/databricks-sdk-go/service/ml" @@ -61,3 +62,41 @@ func TestMaterializationSetMemberKeyEscapesSeparator(t *testing.T) { }) assert.NotEqual(t, first, forged) } + +// operationStub stands in for a generated operation interface. The three verbs' interfaces differ +// only in what Wait returns, which materializationSetOperationTarget never calls. +type operationStub struct { + name string + metadata *ml.MaterializationSetOperationMetadata + err error +} + +func (s operationStub) Name() string { return s.name } + +func (s operationStub) Metadata() (*ml.MaterializationSetOperationMetadata, error) { + return s.metadata, s.err +} + +// The set id names what a failed create, update or delete was changing. The server assigns it +// before reserving the operation record, so the fallback to the operation name covers the cases +// where the metadata is unreadable rather than a set that has no id. +func TestMaterializationSetOperationTarget(t *testing.T) { + assert.Equal(t, "materialization set set-1", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + metadata: &ml.MaterializationSetOperationMetadata{MaterializationSetId: "set-1"}, + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + metadata: &ml.MaterializationSetOperationMetadata{}, + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + err: errors.New("metadata is not a MaterializationSetOperationMetadata"), + })) +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 53fb2da3b38..f93e38067bb 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -865,7 +865,7 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.MaterializationSetsUpdate(req, req.Vars["materialization_set_id"]) }) server.Handle("DELETE", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.MaterializationSets, req.Vars["materialization_set_id"]) + return req.Workspace.MaterializationSetsDelete(req.Vars["materialization_set_id"]) }) // Model Provider Services (AI Gateway): diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index d6aeceefeac..413bfc8f61d 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -222,10 +222,7 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { var set ml.MaterializationSet if err := json.Unmarshal(req.Body, &set); err != nil { - return Response{ - Body: fmt.Sprintf("internal error: %s", err), - StatusCode: http.StatusInternalServerError, - } + return materializationSetInternalError(err) } if len(set.MaterializedFeatures) == 0 { @@ -249,40 +246,29 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { s.MaterializationSets[set.MaterializationSetId] = set - operation, err := s.recordMaterializationSetCreation(set) - if err != nil { - return Response{ - Body: fmt.Sprintf("internal error: %s", err), - StatusCode: http.StatusInternalServerError, - } - } - return Response{Body: operation} + return s.materializationSetOperationResponse(set.MaterializationSetId, set.MaterializedFeatures, set) } -// recordMaterializationSetCreation stores the operation the create returns and hands back a copy. -// The work is finished by the time the response is written, so the operation is recorded already -// terminal; a caller still has to poll it, because the created set is reachable only from there. -func (s *FakeWorkspace) recordMaterializationSetCreation(set ml.MaterializationSet) (ml.Operation, error) { - featureNames := map[string]bool{} - var featureOrder []string - for _, member := range set.MaterializedFeatures { - if !featureNames[member.FeatureName] { - featureNames[member.FeatureName] = true - featureOrder = append(featureOrder, member.FeatureName) - } - } - - metadata, err := json.Marshal(ml.MaterializationSetCreationMetadata{ - FeatureFullNames: featureOrder, - MaterializationSetId: set.MaterializationSetId, - State: ml.MaterializationSetCreationMetadataStateSucceeded, +// materializationSetOperationResponse records the terminal operation a set change returns and +// renders it as the endpoint's response. The work is finished by the time the response is written, +// so the operation is recorded already terminal; a caller still has to poll it, because a create's +// or update's resulting set is reachable only from there. +func (s *FakeWorkspace) materializationSetOperationResponse( + setID string, + members []ml.MaterializedFeature, + result any, +) Response { + metadata, err := json.Marshal(ml.MaterializationSetOperationMetadata{ + FeatureFullNames: materializationSetFeatureNames(members), + MaterializationSetId: setID, + State: ml.MaterializationSetOperationMetadataStateSucceeded, }) if err != nil { - return ml.Operation{}, err + return materializationSetInternalError(err) } - response, err := json.Marshal(set) + response, err := json.Marshal(result) if err != nil { - return ml.Operation{}, err + return materializationSetInternalError(err) } operation := ml.Operation{ @@ -292,7 +278,49 @@ func (s *FakeWorkspace) recordMaterializationSetCreation(set ml.MaterializationS Response: response, } s.Operations[operation.Name] = operation - return operation, nil + return Response{Body: operation} +} + +// materializationSetFeatureNames lists each member's feature once in member order, which is what +// the operation metadata reports. +func materializationSetFeatureNames(members []ml.MaterializedFeature) []string { + seen := map[string]bool{} + var names []string + for _, member := range members { + if !seen[member.FeatureName] { + seen[member.FeatureName] = true + names = append(names, member.FeatureName) + } + } + return names +} + +func materializationSetInternalError(err error) Response { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } +} + +// materializationSetNotFound refuses a set id no set holds. +func materializationSetNotFound(setID string) Response { + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Materialization set with ID '%s' does not exist.", setID)) +} + +// MaterializationSetsDelete fakes DELETE +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. The delete returns a +// long-running operation too, settling with an empty response. +func (s *FakeWorkspace) MaterializationSetsDelete(setID string) Response { + defer s.LockUnlock()() + + existing, ok := s.MaterializationSets[setID] + if !ok { + return materializationSetNotFound(setID) + } + delete(s.MaterializationSets, setID) + + return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, struct{}{}) } // OperationsGet fakes GET /api/2.0/feature-engineering/operations/{operation_id}. @@ -384,16 +412,12 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res existing, ok := s.MaterializationSets[setID] if !ok { - return materializationSetError(http.StatusNotFound, "NOT_FOUND", - fmt.Sprintf("Materialization set with ID '%s' does not exist.", setID)) + return materializationSetNotFound(setID) } var requested ml.MaterializationSet if err := json.Unmarshal(req.Body, &requested); err != nil { - return Response{ - Body: fmt.Sprintf("internal error: %s", err), - StatusCode: http.StatusInternalServerError, - } + return materializationSetInternalError(err) } if len(requested.MaterializedFeatures) == 0 { @@ -466,5 +490,6 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res existing.MaterializedFeatures = matched s.MaterializationSets[setID] = existing - return Response{Body: existing} + + return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, existing) } From ea717a49ac68026b952619c6fe11869ceb0701df Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Tue, 20 Oct 2026 20:21:02 +0000 Subject: [PATCH 18/47] [FS-2621] Match the fake's refusal messages to the reworked set update The set update's validations moved into MaterializationSetUtils and the cascade refusal was reworded to name the removed members of the offending feature, so the fake groups removed members by feature and renders the ids the way the server does. materializedFeatureFieldCannotBeChanged follows the server's new wording too, though no case reaches it. Verified against the daniel-bai-fs2680 swap at universe 8438b2287fb03, where the update no longer refuses a member whose pipeline_schedule_state the config leaves unset. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 50 ++++++++++++ .../membership/output.txt | 8 +- libs/testserver/materialization_set.go | 76 ++++++++++++------- 3 files changed, 103 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index e0e7428221b..5374468743e 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -114,6 +114,56 @@ resources and reference them as `${resources.features..full_name}`, which a feature before the set -- a bare name leaves the create racing the feature, which is what made `sawtooth` flaky. +## Cloud runs 2026-10-20 + +Run against the `daniel-bai-fs2680` swap on workspace 2978662807315490. Two swap builds, and the +difference between them is the point. + +**First build, universe `afc23c250043d`.** `batch`, `sawtooth`, `streaming` and `immutable` passed; +`membership` and `removal_scope` failed, both refused with: + +``` +The fields 'pipeline_schedule_state' of materialized feature '' of feature '' cannot be +changed. Only removing members is supported; recreate the materialization set instead. +``` + +`comparableMaterializationSetMembers` copied the requested `pipelineScheduleState` raw onto the +comparison baseline while normalising the trigger beside it, so a bundle -- which never declares +the field, it being a backend default -- compared `None` against the stored `ACTIVE` and had every +membership change refused. Isolated live: the same removal succeeded when the surviving member +carried `pipeline_schedule_state: ACTIVE` and failed when it did not, nothing else different. + +**Second build, universe `8438b2287fb03`.** Fixed, and the validations moved into +`MaterializationSetUtils`. An omitted state now falls back to the stored one +(`.orElse(normalizedExistingMaterializedFeature.pipelineScheduleState)`), and the changed-field list +is derived from `MaterializedFeature.scalaDescriptor.fields` rather than a hand-maintained `Seq`, so +a member field added later is covered without touching this check. + +All six cases have passed against that build. The one golden this cost was `membership`: the cascade +refusal was reworded, and the fake carried the old text. + +``` +old: Changing one materialized feature of feature 'F' also changes '', which the request does + not change. Apply the same change to every materialized feature of the feature. +new: Changing materialized features '' of feature 'F' also changes materialized features + '', which the request does not change. Apply the same change to the unrequested + materialized features as well. +``` + +The new message names the removed members of the offending feature, so the fake groups removed +members by feature and sorts the ids the way `FeatureStoreMessages.quotedIds` renders them. +`materializedFeatureFieldCannotBeChanged` was reworded to match the server too, though no case +reaches it -- the planner classifies every member field as a recreate, so a declared change never +becomes an update request. + +**The workspace throws intermittent 500s that are not this resource.** Across five runs of the +`DMS=` leg, individual cases failed on `POST /api/2.0/feature-engineering/features` with `Unable to +create Feature '' in UC. If the issue persists, please retry or contact support.`, and once on +`Failed to create Lakebase table`. `immutable` lost three runs in a row this way. It is not the +fixture: its two features are identical but for their names, and the one that fails varies. It did +not reproduce from a direct two-feature create, serial or concurrent. Budget a retry per case rather +than reading a single red run as a regression. + ## Verified against a workspace, with no case of their own Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index c7215b0d753..2c217d8e132 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -16,12 +16,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === refused, because the delete also reaches the offline member the config still declares >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. +API message: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. Files: 1 uploaded, 0 deleted @@ -29,12 +29,12 @@ Files: 1 uploaded, 0 deleted === direction >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Changing one materialized feature of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes '[UUID]', which the request does not change. Apply the same change to every materialized feature of the feature. +API message: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. Files: 1 uploaded, 0 deleted diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 413bfc8f61d..8054c36fec5 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "sort" "strings" "github.com/databricks/databricks-sdk-go/service/ml" @@ -24,12 +25,25 @@ func materializationSetError(statusCode int, errorCode, message string) Response } } -// materializedFeatureFieldCannotBeChanged refuses a field the set update cannot write. Every -// member field is one, so the only change the endpoint applies is a removal. -func materializedFeatureFieldCannotBeChanged(featureName, fieldPath string) Response { +// quotedMaterializedFeatureIDs renders an id list the way FeatureStoreMessages.quotedIds does. +func quotedMaterializedFeatureIDs(ids []string) string { + quoted := make([]string, len(ids)) + for i, id := range ids { + quoted[i] = "'" + id + "'" + } + return strings.Join(quoted, ", ") +} + +// materializedFeatureFieldCannotBeChanged refuses a field the set update cannot write. No +// acceptance case reaches it: the planner classifies every member field as a recreate, so a +// declared change never becomes an update request. +func materializedFeatureFieldCannotBeChanged( + featureName, materializedFeatureID, fieldPath string, +) Response { return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("The '%s' of feature '%s' cannot be changed after its materialized features "+ - "are created. Recreate the materialization set instead.", fieldPath, featureName)) + fmt.Sprintf("The fields '%s' of materialized feature '%s' of feature '%s' cannot be "+ + "changed. Only removing members is supported; recreate the materialization set "+ + "instead.", fieldPath, materializedFeatureID, featureName)) } // materializationSetMemberKey matches a requested member to a stored one the way the backend does: @@ -445,13 +459,14 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res } if isCronScheduleChangeRequested(member.CronScheduleTrigger, stored.CronScheduleTrigger) { return materializedFeatureFieldCannotBeChanged(member.FeatureName, - "cron_schedule_trigger") + stored.MaterializedFeatureId, "trigger") } - // An unset state asks for no change, so the comparison is only against a stated one. + // An unset state asks for no change: the server falls back to the stored one before + // comparing, so a member the config never declares is not read as a change. if member.PipelineScheduleState != "" && member.PipelineScheduleState != stored.PipelineScheduleState { return materializedFeatureFieldCannotBeChanged(member.FeatureName, - "pipeline_schedule_state") + stored.MaterializedFeatureId, "pipeline_schedule_state") } requestedKeys[materializationSetMemberKey(member)] = true matched = append(matched, stored) @@ -459,33 +474,40 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res // Delete the members the request omits. Because a delete cascades from a feature's offline // member to its online ones, a removal that would also reach a member the request still lists - // cannot be expressed and is refused before anything is written. + // cannot be expressed and is refused before anything is written. The server groups the removed + // members by feature and names that feature's removed ids, so the message is built the same way. removedIDs := map[string]bool{} + featureNameToRemovedIDs := map[string][]string{} for _, member := range existing.MaterializedFeatures { if !requestedKeys[materializationSetMemberKey(member)] { removedIDs[member.MaterializedFeatureId] = true + featureNameToRemovedIDs[member.FeatureName] = append( + featureNameToRemovedIDs[member.FeatureName], member.MaterializedFeatureId) } } - if len(removedIDs) > 0 { - for _, target := range materializationSetDeletionTargets(existing.MaterializedFeatures) { - var unrequested []string - requestedForChange := false - for _, member := range target { - if removedIDs[member.MaterializedFeatureId] { - requestedForChange = true - } else { - unrequested = append(unrequested, "'"+member.MaterializedFeatureId+"'") - } - } - // A target the caller asked nothing about is not in play. - if requestedForChange && len(unrequested) > 0 { - return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("Changing one materialized feature of feature '%s' also changes "+ - "%s, which the request does not change. Apply the same change to every "+ - "materialized feature of the feature.", - target[0].FeatureName, strings.Join(unrequested, ", "))) + for _, target := range materializationSetDeletionTargets(existing.MaterializedFeatures) { + var unrequested []string + requestedForChange := false + for _, member := range target { + if removedIDs[member.MaterializedFeatureId] { + requestedForChange = true + } else { + unrequested = append(unrequested, member.MaterializedFeatureId) } } + // A target the caller asked nothing about is not in play. + if !requestedForChange || len(unrequested) == 0 { + continue + } + featureName := target[0].FeatureName + removedForFeature := append([]string(nil), featureNameToRemovedIDs[featureName]...) + sort.Strings(removedForFeature) + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Changing materialized features %s of feature '%s' also changes "+ + "materialized features %s, which the request does not change. Apply the same "+ + "change to the unrequested materialized features as well.", + quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested))) } existing.MaterializedFeatures = matched From 32fe896ef79d29bc2a5d4cf85309c68bb4fccb07 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Tue, 20 Oct 2026 20:22:38 +0000 Subject: [PATCH 19/47] [FS-2621] Record that the cloud 500s track parallel load, not the fixture Co-authored-by: Isaac --- .../bundle/resources/materialization_sets/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 5374468743e..c0b23c7e628 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -159,10 +159,11 @@ becomes an update request. **The workspace throws intermittent 500s that are not this resource.** Across five runs of the `DMS=` leg, individual cases failed on `POST /api/2.0/feature-engineering/features` with `Unable to create Feature '' in UC. If the issue persists, please retry or contact support.`, and once on -`Failed to create Lakebase table`. `immutable` lost three runs in a row this way. It is not the -fixture: its two features are identical but for their names, and the one that fails varies. It did -not reproduce from a direct two-feature create, serial or concurrent. Budget a retry per case rather -than reading a single red run as a regression. +`Failed to create Lakebase table`. `immutable` lost three runs in a row this way, then passed twice in +a row when run on its own -- so the trigger is the parallel load of six bundles deploying at once, +not the fixture. Its two features are identical but for their names, and which one fails varies; a +direct two-feature create reproduced nothing, serial or concurrent. Run a case on its own before +reading a red run as a regression. ## Verified against a workspace, with no case of their own From 32a4fd035c3d7051e3f9d3f9b0487541b26054a3 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Tue, 20 Oct 2026 22:13:50 +0000 Subject: [PATCH 20/47] [FS-2621] Restate a partly applied set change's failed members in the error The update and delete endpoints name the members they could not reach in a MaterializationSet*Progress error detail and leave the message deferring to it. libs/diag/sdk_error.go renders only error_code and message, and nothing in the bundle path reads an error detail, so those ids never reached the user. Read from APIError.ErrorDetails().UnknownDetails, which is where the SDK puts a detail type it does not model. Applied to the wait as well as the call, so it holds whether the server raises the partial failure or settles the operation with it. Co-authored-by: Isaac --- .../direct/dresources/materialization_set.go | 75 ++++++++++++++++++- .../dresources/materialization_set_test.go | 75 +++++++++++++++++++ 2 files changed, 146 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 62b701b0205..090ee8f9be7 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -2,7 +2,9 @@ package dresources import ( "context" + "errors" "fmt" + "slices" "strconv" "strings" @@ -10,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/google/uuid" @@ -169,6 +172,68 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma return set.MaterializationSetId, set, nil } +// materializationSetProgressTypes name the error details a partly applied set change carries, one +// per verb that can partly apply. A creation cannot: it has no per-member outcome. +var materializationSetProgressTypes = []string{ + "type.googleapis.com/databricks.featurestore.MaterializationSetUpdateProgress", + "type.googleapis.com/databricks.featurestore.MaterializationSetDeletionProgress", +} + +// protoJSONField reads a proto field out of a decoded JSON object under either spelling a +// serializer may emit for it. +func protoJSONField(fields map[string]any, snakeCase, camelCase string) any { + if value, present := fields[snakeCase]; present { + return value + } + return fields[camelCase] +} + +// materializationSetChangeFailures names each member a partly applied change could not reach, +// taken from the progress the server attaches as an error detail. Empty for an error raised before +// the server touched a member, which is every validation failure. +func materializationSetChangeFailures(err error) []string { + apiErr, isAPIError := errors.AsType[*apierr.APIError](err) + if !isAPIError || apiErr == nil { + return nil + } + + var failures []string + for _, detail := range apiErr.ErrorDetails().UnknownDetails { + fields, isObject := detail.(map[string]any) + if !isObject { + continue + } + typeURL, _ := fields["@type"].(string) + if !slices.Contains(materializationSetProgressTypes, typeURL) { + continue + } + entries, _ := protoJSONField( + fields, "failed_materialized_features", "failedMaterializedFeatures").([]any) + for _, entry := range entries { + failure, isObject := entry.(map[string]any) + if !isObject { + continue + } + id, _ := protoJSONField(failure, "materialized_feature_id", "materializedFeatureId").(string) + errorCode, _ := protoJSONField(failure, "error_code", "errorCode").(string) + message, _ := failure["message"].(string) + failures = append(failures, fmt.Sprintf("%s (%s): %s", id, errorCode, message)) + } + } + return failures +} + +// withMaterializationSetChangeFailures restates the members a partly applied change left alone in +// the error itself. Applied to the wait as well as the call, so it holds whether the server raises +// the partial failure or settles the operation with it. +func withMaterializationSetChangeFailures(err error) error { + failures := materializationSetChangeFailures(err) + if len(failures) == 0 { + return err + } + return fmt.Errorf("%w; members left unchanged: %s", err, strings.Join(failures, "; ")) +} + // materializationSetOperation is what a create, update and delete operation have in common: the // three generated interfaces share these two methods and differ only in what Wait returns. type materializationSetOperation interface { @@ -199,12 +264,13 @@ func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, co RequestId: uuid.NewString(), }) if err != nil { - return nil, err + return nil, withMaterializationSetChangeFailures(err) } set, err := operation.Wait(ctx) if err != nil { - return nil, fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), err) + return nil, fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), + withMaterializationSetChangeFailures(err)) } return set, nil } @@ -217,10 +283,11 @@ func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ RequestId: uuid.NewString(), }) if err != nil { - return err + return withMaterializationSetChangeFailures(err) } if err := operation.Wait(ctx); err != nil { - return fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), err) + return fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), + withMaterializationSetChangeFailures(err)) } return nil } diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go index 8458ee2be14..41aea2e8a8a 100644 --- a/bundle/direct/dresources/materialization_set_test.go +++ b/bundle/direct/dresources/materialization_set_test.go @@ -1,11 +1,20 @@ package dresources import ( + "bytes" + "context" + "encoding/json" "errors" + "io" + "net/http" + "net/url" "testing" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMaterializationSetMemberKey(t *testing.T) { @@ -100,3 +109,69 @@ func TestMaterializationSetOperationTarget(t *testing.T) { err: errors.New("metadata is not a MaterializationSetOperationMetadata"), })) } + +// apiErrorWithDetails builds the error the SDK produces from a response body carrying error +// details, which is the only way the per-member outcome of a partly applied change reaches a +// client. +func apiErrorWithDetails(t *testing.T, message string, details []map[string]any) error { + t.Helper() + body, err := json.Marshal(map[string]any{ + "error_code": "RESOURCE_CONFLICT", + "message": message, + "details": details, + }) + require.NoError(t, err) + + resp := &http.Response{ + StatusCode: http.StatusConflict, + Status: "409 Conflict", + Request: &http.Request{Method: http.MethodPatch, URL: &url.URL{Path: "/api/2.0/x"}}, + } + return apierr.GetAPIError(context.Background(), common.ResponseWrapper{ + Response: resp, + DebugBytes: body, + ReadCloser: io.NopCloser(bytes.NewReader(body)), + }) +} + +func updateProgressDetail(failures []map[string]any) map[string]any { + return map[string]any{ + "@type": "type.googleapis.com/databricks.featurestore.MaterializationSetUpdateProgress", + "deleted_materialized_feature_ids": []any{"mf-gone"}, + "failed_materialized_features": failures, + } +} + +// The server's message defers to an error detail the CLI never prints, so the adapter has to +// restate the failed members in the error itself. +func TestMaterializationSetChangeFailuresAreRestatedInTheError(t *testing.T) { + err := apiErrorWithDetails(t, + "Changed 1 of 2 materialized features in materialization set 'set-1'; see the error "+ + "details for which ones failed.", + []map[string]any{updateProgressDetail([]map[string]any{{ + "materialized_feature_id": "mf-stuck", + "error_code": "RESOURCE_CONFLICT", + "message": "pipeline is updating", + }})}) + + wrapped := withMaterializationSetChangeFailures(err) + assert.Contains(t, wrapped.Error(), "members left unchanged:") + assert.Contains(t, wrapped.Error(), "mf-stuck (RESOURCE_CONFLICT): pipeline is updating") + // The server's own message survives, so nothing a caller already relied on is lost. + assert.Contains(t, wrapped.Error(), "Changed 1 of 2 materialized features") + assert.ErrorIs(t, wrapped, err) +} + +// Every validation failure is raised before the server touches a member, so it carries no progress +// and must come back exactly as it arrived. +func TestMaterializationSetChangeFailuresLeavesOtherErrorsAlone(t *testing.T) { + withoutDetails := apiErrorWithDetails(t, "Materialization set 'set-1' does not exist.", nil) + assert.Equal(t, withoutDetails, withMaterializationSetChangeFailures(withoutDetails)) + + // A progress detail naming no failed member is a succeeded change, not a partial one. + empty := apiErrorWithDetails(t, "transient", []map[string]any{updateProgressDetail(nil)}) + assert.Equal(t, empty, withMaterializationSetChangeFailures(empty)) + + plain := errors.New("connection reset") + assert.Equal(t, plain, withMaterializationSetChangeFailures(plain)) +} From 31ce4a33ed3df17bf4703d8ff4c54fd15d78b318 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Wed, 21 Oct 2026 16:57:09 +0000 Subject: [PATCH 21/47] [FS-2621] Point the member key at the server class it duplicates The adapter's member key and MaterializationSetMemberKey.keyOf derive the same three components in the same order, and the server pairs a requested member with a stored one by it, so the two have to move together. The comment named neither the class nor that coupling. Co-authored-by: Isaac --- bundle/direct/dresources/materialization_set.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 090ee8f9be7..daee8567f9b 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -47,6 +47,12 @@ func escapeMemberKeyPart(part string) string { // materializationSetMemberKey identifies a materialized feature member by content rather than // position. The key components are the feature name, the online store name (empty for the offline // member), and whether the trigger is streaming. +// +// The server builds a key from the same three components, in the same order, in +// MaterializationSetMemberKey.keyOf -- see +// feature-store/src/main/scala/com/databricks/feature-store/utils/materialized-features/MaterializationSetUtils.scala +// -- and pairs a requested member with a stored one by it. Adding, dropping or redefining a +// component here takes the same change there. func materializationSetMemberKey(member ml.MaterializedFeature) (string, string) { onlineStoreName := "" if member.OnlineStoreConfig != nil { From 8ee470cbec05b213f0b8467ce129053b2937e31a Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Wed, 21 Oct 2026 17:50:20 +0000 Subject: [PATCH 22/47] [FS-2621] Pin the whole removal matrix, and the two refusals the fake was missing The backend now has a message for each direction of the deletion cascade, picked by whether the removal names the member a delete is applied to, and refuses a set that mixes a batch-path feature with a streaming-path one. The fake carried one older cascade message for both directions and did not model the mixing rule at all. The cases now walk the removal matrix rather than sampling it: sawtooth covers all three strict subsets of its one cascade group, batch covers each half of its pair, streaming gains a second feature so a lone online member can be removed as an update, and immutable covers the destroy and redeploy pair those refusals leave as the only route. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 25 +++++- .../batch/drop-both.yml.tmpl | 26 +++++++ .../batch/drop-offline.yml.tmpl | 35 +++++++++ .../batch/drop-online.yml.tmpl | 36 +++++++++ .../materialization_sets/batch/mixed.yml.tmpl | 63 +++++++++++++++ .../materialization_sets/batch/output.txt | 76 +++++++++++++++++- .../materialization_sets/batch/script | 28 +++++++ .../materialization_sets/immutable/output.txt | 35 +++++++++ .../materialization_sets/immutable/script | 11 +++ .../membership/output.txt | 8 +- .../sawtooth/drop-all.yml.tmpl | 25 ++++++ .../sawtooth/drop-batch-pair.yml.tmpl | 34 ++++++++ .../sawtooth/drop-offline.yml.tmpl | 43 ++++++++++ .../sawtooth/drop-online-batch.yml.tmpl | 42 ++++++++++ .../materialization_sets/sawtooth/output.txt | 78 ++++++++++++++++++- .../materialization_sets/sawtooth/script | 39 ++++++++++ .../streaming/databricks.yml.tmpl | 28 +++++++ .../streaming/drop-one.yml.tmpl | 55 +++++++++++++ .../streaming/moved.yml.tmpl | 64 +++++++++++++++ .../materialization_sets/streaming/output.txt | 33 +++++++- .../materialization_sets/streaming/script | 25 ++++++ libs/testserver/materialization_set.go | 59 +++++++++++++- 22 files changed, 852 insertions(+), 16 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index c0b23c7e628..df637d5a9de 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -38,11 +38,11 @@ because every adapter call mints its own. | `membership` | the delete cascade runs both ways -- removing either member of a feature's pair on its own is refused, while removing a whole pair plans as `update` and the other feature's pair keeps its ids | | `removal_scope` | the cascade is scoped per feature: where every feature is materialized offline only, each contributes one member and can have it removed alone; the update's operation round trip | | `schedule_state` | a member the config leaves unset adopts the state the server assigned; a declared change to it is a `recreate`, and that recreate still has to satisfy the create endpoint's rule that a feature's members agree | -| `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name | +| `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name; destroy then redeploy is the route those refusals leave open, and the re-created set plans clean | | `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | -| `batch` | a batch feature with both destinations creates and settles; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not | -| `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing | -| `streaming` | a lone streaming online member needs no schedule and no cron; declaring a schedule state on it plans as a `recreate` rather than an in-place update | +| `batch` | a batch feature with both destinations creates and settles; neither member of its pair can be removed alone, and removing both empties the set, which the update endpoint refuses in favour of delete; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not; a set mixing a batch-path feature with a streaming-path one is refused at create | +| `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing; the three members are one cascade group, so each of the three strict subsets of them is refused -- with the message depending on whether the removal names the member a delete is applied to -- and removing all three empties the set, which the update endpoint refuses | +| `streaming` | a streaming online member needs no schedule and no cron; declaring a schedule state on it, or repointing its destination, plans as a `recreate` rather than an in-place update; with two features materialized to one destination and no offline member between them, each online member is its own cascade group, so one can be removed alone as an `update` | ## Running against a workspace @@ -165,6 +165,23 @@ not the fixture. Its two features are identical but for their names, and which o direct two-feature create reproduced nothing, serial or concurrent. Run a case on its own before reading a red run as a regression. +**Third build, universe `275193b7facc8`.** The deletion cascade refusal became two messages, +picked by whether the removal names the member a delete is applied to: +`materializationSetMembersCannotBeRemovedAlone` when it does not, and +`deletionAffectsUnrequestedMaterializedFeatureMembers` when it does. The fake branches the same way +and `membership`'s two arms now show one each. A set mixing a batch-path feature with a +streaming-path one is also refused, which the fake had not modelled; `batch`'s last arm pins it, +with the message captured from this build. + +**The removal-matrix arms added for this build have not run against a workspace.** Every message in +them came from the backend source at `275193b7facc8` or, for the mixing refusal and the +members-cannot-be-removed-alone wording, from a live deploy against the swap. The arms themselves -- +sawtooth's three subsets, batch's pair, streaming's second feature, immutable's destroy and +redeploy -- have only run against the fake. The `streaming` case's claim that two stream features +can share one set is the one that most needs checking: it rests on the shared-destination rule +comparing a `(offline, Option[online])` pair that is identical for both, which is read off the +source rather than observed. + ## Verified against a workspace, with no case of their own Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl new file mode 100644 index 00000000000..234b08d05e6 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: [] diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl new file mode 100644 index 00000000000..587d203b3bc --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl @@ -0,0 +1,35 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl new file mode 100644 index 00000000000..032124311b0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl @@ -0,0 +1,36 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl new file mode 100644 index 00000000000..9bba1ce5161 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl @@ -0,0 +1,63 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + # Stream-sourced, so it materializes on the streaming path while batchfeat stays on the + # batch path. The set below asks for one of each. + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batchstream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batchstream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index faabce600c7..9aad1a0ef30 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -9,7 +9,7 @@ Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Created features.batchfeat Created materialization_sets.set1 -Files: 5 uploaded, 0 deleted +Files: 9 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json @@ -19,6 +19,65 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "resources.materialization_sets.set1": "skip" } +=== Neither member of the pair can be removed on its own. The offline member is the one a +=== delete is applied to, so removing it reaches the online one too, +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== and removing the online member names nothing a delete can be applied to +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. + +Files: 2 uploaded, 0 deleted + +=== Removing both satisfies the cascade but empties the set, which the update endpoint refuses +=== in favour of the delete endpoint. Restoring the pair plans nothing, so no refusal wrote. +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 1 uploaded, 0 deleted + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + === Repointing the destinations is a recreate, not a silent no-op. The server cannot move a === materialization in place, so the set is rebuilt at the new location. >>> [CLI] bundle plan -o json @@ -83,3 +142,18 @@ API error_code: INVALID_PARAMETER_VALUE API message: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. Files: 2 uploaded, 0 deleted + +=== A set cannot mix a feature that materializes on the batch path with one that materializes +=== on the streaming path. The refusal above left no set behind, so this reaches the create +=== endpoint, and it is raised before the shared-destination rule the two features also break. +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created features.streamfeat +Error: cannot create resources.materialization_sets.set1: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script index 7650bbe0d1e..f7e135e497c 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/script +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -1,6 +1,7 @@ export CATALOG_NAME="${MATSET_CATALOG:-danielb}" export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" # Each phase is its own template rather than a string edit, so the fixture a phase deploys is @@ -29,6 +30,27 @@ trace $CLI bundle deploy trace $CLI bundle plan -o json > out.plan.json trace set_actions +title "Neither member of the pair can be removed on its own. The offline member is the one a" +title "delete is applied to, so removing it reaches the online one too," +render drop-offline.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "and removing the online member names nothing a delete can be applied to" +render drop-online.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "Removing both satisfies the cascade but empties the set, which the update endpoint refuses" +title "in favour of the delete endpoint. Restoring the pair plans nothing, so no refusal wrote." +render drop-both.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + title "Repointing the destinations is a recreate, not a silent no-op. The server cannot move a" title "materialization in place, so the set is rebuilt at the new location." render moved.yml.tmpl @@ -52,4 +74,10 @@ trace $CLI bundle plan -o json > out.plan.json trace set_actions trace musterr $CLI bundle deploy --auto-approve +title "A set cannot mix a feature that materializes on the batch path with one that materializes" +title "on the streaming path. The refusal above left no set behind, so this reaches the create" +title "endpoint, and it is raised before the shared-destination rule the two features also break." +render mixed.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt index 9d2ae096477..21ed2a4d4e4 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/output.txt +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -40,3 +40,38 @@ API error_code: INVALID_PARAMETER_VALUE API message: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. Files: 2 uploaded, 0 deleted + +=== Every refusal above leaves destroy and redeploy as the only route, so that pair is what a +=== member-field change costs. The delete settles its own operation, and the re-created set +=== plans clean. +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.feata + delete resources.features.featb + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default + +Destroy: 3 deleted + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "create" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "skip" +} diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script index 08de9633a63..99138f35364 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/script +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -38,4 +38,15 @@ trace $CLI bundle plan -o json > out.plan.json trace set_action trace musterr $CLI bundle deploy --auto-approve +title "Every refusal above leaves destroy and redeploy as the only route, so that pair is what a" +title "member-field change costs. The delete settles its own operation, and the re-created set" +title "plans clean." +render databricks.yml.tmpl +trace $CLI bundle destroy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_action + rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index 2c217d8e132..c7b7620a49f 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -16,12 +16,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === refused, because the delete also reaches the offline member the config still declares >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. +API message: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. Files: 1 uploaded, 0 deleted @@ -29,12 +29,12 @@ Files: 1 uploaded, 0 deleted === direction >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Changing materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also changes materialized features '[UUID]', which the request does not change. Apply the same change to the unrequested materialized features as well. +API message: Deleting materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl new file mode 100644 index 00000000000..34db72c23f8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: [] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl new file mode 100644 index 00000000000..cfae25e8a3b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl new file mode 100644 index 00000000000..9f86ac53015 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl @@ -0,0 +1,43 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl new file mode 100644 index 00000000000..3969ab6e672 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt index d8a814027f8..e5ac092c9ea 100644 --- a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -13,7 +13,7 @@ HTTP Status: 400 Bad Request API error_code: BAD_REQUEST API message: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. -Files: 3 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted === Declaring all three creates the set >>> [CLI] bundle deploy @@ -39,3 +39,79 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === Every suppressed destination change, by reason >>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length out.plan.json 7 + +=== A sawtooth feature's three members are one cascade group, so every strict subset of them +=== is refused. Removing the offline member reaches both online ones, +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== removing the online batch member names no member a delete can be applied to, +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request doesn't supply. Remove '[UUID]', '[UUID]' as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request doesn't supply. Remove '[UUID]', '[UUID]' as well. + +Files: 2 uploaded, 0 deleted + +=== and removing the batch pair still leaves the streaming member behind +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]', '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]', '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== Removing all three satisfies the cascade but empties the set, which the update endpoint +=== refuses in favour of the delete endpoint. None of the four refusals wrote anything, so +=== restoring the config plans nothing. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle plan -o json + +>>> set_action +skip diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/script b/acceptance/bundle/resources/materialization_sets/sawtooth/script index d44a778b64b..0184d2f2f84 100644 --- a/acceptance/bundle/resources/materialization_sets/sawtooth/script +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/script @@ -13,6 +13,15 @@ pin_profile() { printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml } +render() { + envsubst < "$1" > databricks.yml + pin_profile +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + cleanup() { $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true } @@ -40,4 +49,34 @@ trace $CLI bundle deploy title "Every suppressed destination change, by reason" trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length' out.plan.json +title "A sawtooth feature's three members are one cascade group, so every strict subset of them" +title "is refused. Removing the offline member reaches both online ones," +render drop-offline.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "removing the online batch member names no member a delete can be applied to," +render drop-online-batch.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "and removing the batch pair still leaves the streaming member behind" +render drop-batch-pair.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Removing all three satisfies the cascade but empties the set, which the update endpoint" +title "refuses in favour of the delete endpoint. None of the four refusals wrote anything, so" +title "restoring the config plans nothing." +render drop-all.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl index 8ab5d8b17e3..5e09efb43e1 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl @@ -3,6 +3,25 @@ bundle: resources: features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + streamfeat: full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME source: @@ -34,3 +53,12 @@ resources: table_name_prefix: stream_$UNIQUE_NAME streaming_mode: mode: STREAMING_MODE_TYPE_RTM + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat2.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl new file mode 100644 index 00000000000..a425fee6857 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl @@ -0,0 +1,55 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl new file mode 100644 index 00000000000..005b9a122ed --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl @@ -0,0 +1,64 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_${UNIQUE_NAME}_moved + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat2.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/output.txt b/acceptance/bundle/resources/materialization_sets/streaming/output.txt index d6959ee5a0a..b9ee84b6ae9 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/output.txt +++ b/acceptance/bundle/resources/materialization_sets/streaming/output.txt @@ -3,9 +3,10 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... Created features.streamfeat +Created features.streamfeat2 Created materialization_sets.set1 -Files: 2 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === A redeploy plans nothing. The schedule state the server assigns to a lone streaming member === must not read as drift, and neither must its online destination. @@ -19,7 +20,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... Files: 1 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged === The schedule state is write-once, so declaring one on a streaming member that had none === plans as a recreate rather than an in-place update @@ -32,3 +33,29 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged { "resources.materialization_sets.set1": "recreate" } + +=== Repointing a member's online destination is a recreate: the server cannot move a +=== materialization in place, so the whole set is rebuilt at the new location +>>> [CLI] bundle plan -o json + +>>> set_action +recreate + +=== Removing one feature's streaming member is an update the endpoint accepts. With no offline +=== member to cascade from, each online member is its own cascade group, so the other feature's +=== member is untouched and the set is left non-empty. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +skip diff --git a/acceptance/bundle/resources/materialization_sets/streaming/script b/acceptance/bundle/resources/materialization_sets/streaming/script index bf167bdaf04..16a98ab9394 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/script +++ b/acceptance/bundle/resources/materialization_sets/streaming/script @@ -12,6 +12,15 @@ pin_profile() { printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml } +render() { + envsubst < "$1" > databricks.yml + pin_profile +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + cleanup() { $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true } @@ -36,4 +45,20 @@ trace update_file.py databricks.yml ' streaming_mode:' ' pipel trace $CLI bundle plan -o json > out.plan.json trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +title "Repointing a member's online destination is a recreate: the server cannot move a" +title "materialization in place, so the whole set is rebuilt at the new location" +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "Removing one feature's streaming member is an update the endpoint accepts. With no offline" +title "member to cascade from, each online member is its own cascade group, so the other feature's" +title "member is untouched and the set is left non-empty." +render drop-one.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action + rm -f out.requests.txt out.plan.json diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 8054c36fec5..13e02f99672 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -188,6 +188,45 @@ func (s *FakeWorkspace) isSawtoothFeature(featureName string) bool { return aggregation != nil && aggregation.TimeWindow != nil && aggregation.TimeWindow.Sawtooth != nil } +// validateBatchAndStreamingFeaturesNotMixed mirrors the create-path rule that one call materializes +// either batch features or streaming features. A Stream-sourced feature always materializes on the +// streaming path; a Delta-sourced one joins it only when a member of it carries a streaming +// trigger. An unknown feature falls in neither group, for the reason isSawtoothFeature gives. +func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( + members []ml.MaterializedFeature, +) *Response { + var batchFeatureNames, streamingFeatureNames []string + for _, featureName := range materializationSetFeatureNames(members) { + feature, stored := s.Features[featureName] + if !stored { + continue + } + hasStreamingMember := false + for _, member := range members { + if member.FeatureName == featureName && member.StreamingMode != nil { + hasStreamingMember = true + } + } + switch { + case feature.Source.StreamSource != nil || hasStreamingMember: + streamingFeatureNames = append(streamingFeatureNames, featureName) + case feature.Source.DeltaTableSource != nil: + batchFeatureNames = append(batchFeatureNames, featureName) + } + } + if len(batchFeatureNames) == 0 || len(streamingFeatureNames) == 0 { + return nil + } + + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Batch and streaming features cannot be mixed in a single "+ + "materialize_features call because they require different trigger types. Issue two "+ + "separate calls: one for batch features and one for streaming features. Batch "+ + "features: %s. Streaming features: %s", + strings.Join(batchFeatureNames, ", "), strings.Join(streamingFeatureNames, ", "))) + return &response +} + // validateSawtoothMaterializedFeatures mirrors the rule that a sawtooth feature declares all three // of its materialized features itself. The request is not expanded for it: a materialization set // stamps a set id on every member before the backend reaches its normalization, and that @@ -246,6 +285,9 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { return *bad } + if bad := s.validateBatchAndStreamingFeaturesNotMixed(set.MaterializedFeatures); bad != nil { + return *bad + } if bad := validateSharedMaterializationDestination(set.MaterializedFeatures); bad != nil { return *bad } @@ -502,11 +544,22 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res featureName := target[0].FeatureName removedForFeature := append([]string(nil), featureNameToRemovedIDs[featureName]...) sort.Strings(removedForFeature) + // The server has a message for each direction. target[0] is the member a delete is applied + // to, so naming it means the delete reaches past what the request asked for, and omitting + // it means the named members have no delete of their own. + if removedIDs[target[0].MaterializedFeatureId] { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Deleting materialized features %s of feature '%s' also deletes "+ + "materialized features %s, which the request does not delete. Delete the "+ + "unrequested materialized features as well.", + quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested))) + } return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("Changing materialized features %s of feature '%s' also changes "+ - "materialized features %s, which the request does not change. Apply the same "+ - "change to the unrequested materialized features as well.", + fmt.Sprintf("Materialized features %s of feature '%s' are deleted only by deleting "+ + "materialized features %s, which the request doesn't supply. Remove %s as well.", quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested), quotedMaterializedFeatureIDs(unrequested))) } From 85ab9b03d673b894028a83e39e85f1bcba9cfb1a Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Wed, 21 Oct 2026 18:15:17 +0000 Subject: [PATCH 23/47] [FS-2621] Record that the removal matrix passed against a workspace Every message the new arms assert is now the server's own text rather than a reading of its source, including the claim that two stream features materialized to one destination can share a set. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index df637d5a9de..57d76260ccf 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -173,14 +173,12 @@ and `membership`'s two arms now show one each. A set mixing a batch-path feature streaming-path one is also refused, which the fake had not modelled; `batch`'s last arm pins it, with the message captured from this build. -**The removal-matrix arms added for this build have not run against a workspace.** Every message in -them came from the backend source at `275193b7facc8` or, for the mixing refusal and the -members-cannot-be-removed-alone wording, from a live deploy against the swap. The arms themselves -- -sawtooth's three subsets, batch's pair, streaming's second feature, immutable's destroy and -redeploy -- have only run against the fake. The `streaming` case's claim that two stream features -can share one set is the one that most needs checking: it rests on the shared-destination rule -comparing a `(offline, Option[online])` pair that is identical for both, which is read off the -source rather than observed. +All six `Cloud = true` cases pass against this build, run one at a time, with no golden changed -- +so every message in the removal matrix is the server's own text and not a reading of its source. +That covers both cascade messages, the empty-set refusal that sends a caller to the delete endpoint, +the batch-versus-streaming mixing refusal, and the `streaming` case's claim that two stream features +materialized to one destination can share a set, which until then rested on the shared-destination +rule comparing a `(offline, Option[online])` pair that is identical for both. ## Verified against a workspace, with no case of their own From 45caeae6efd96f6416671b9ecc02c9f7bb8086cb Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Thu, 22 Oct 2026 04:58:45 +0000 Subject: [PATCH 24/47] [FS-2621] Read a settled-but-failed operation instead of its error details Once the backend has reserved an operation record it no longer raises: all three verbs settle the record FAILED and return it with HTTP 200, with the error code and message on the operation and the per-member outcome on the metadata's progress arm. The adapter was reading error details off an APIError, which that shape never produces, so a partial change surfaced as a flat string with the members it left alone dropped. The adapter now re-reads the settled operation, recovers the code structurally, appends the failed members, and returns an APIError so the engine classifies and renders it like any other failure. The status is the 200 the call returned, which maps to no sentinel on purpose: a member that could not be torn down must not reach the destroy step's permission-denied leniency. The fake can produce the shape too, keyed off a marker in a feature name, since a real teardown fails on infrastructure no request can ask for. The new settled_failure case pins what the adapter makes of it, and pins the engine gap behind it: a destroy forgives a delete error matching none of its known cases, so a partly deleted set exits zero with its state dropped. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 29 +++- .../settled_failure/databricks.yml.tmpl | 79 +++++++++++ .../settled_failure/drop-failing.yml.tmpl | 62 +++++++++ .../settled_failure/out.test.toml | 2 + .../settled_failure/output.txt | 48 +++++++ .../settled_failure/script | 36 +++++ .../settled_failure/test.toml | 12 ++ .../direct/dresources/materialization_set.go | 111 +++++++--------- .../dresources/materialization_set_test.go | 102 +++++--------- libs/testserver/materialization_set.go | 124 ++++++++++++++++++ 10 files changed, 474 insertions(+), 131 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/settled_failure/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/settled_failure/script create mode 100644 acceptance/bundle/resources/materialization_sets/settled_failure/test.toml diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 57d76260ccf..23b4fe25f1f 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -29,6 +29,25 @@ mints a new token; that is safe here because it never retries a create without t in, and re-running an update or a delete reconciles to the same state. No case replays a token, because every adapter call mints its own. +## A failure after the operation is reserved is not an HTTP error + +The backend raises only while it can still reject the request outright -- every validation, which +runs before it reserves an operation record. Past that point a failure is the operation's outcome, +so all three verbs settle the record FAILED and return it with HTTP 200, carrying the error code and +message on the operation and the per-member outcome on the metadata's progress arm. A creation has +no progress arm: it has no per-member outcome to report. + +The generated wait flattens that into one string and reads no metadata, so the adapter re-reads the +settled operation to recover the code structurally and appends the members the change left alone. +It returns an `APIError` so the engine classifies and renders the failure as it would any other, +with the 200 the call actually returned as the status -- which maps to no sentinel, deliberately: a +member that could not be torn down must not reach the destroy step's permission-denied leniency, +whose purpose is to forgive a resource that is already gone. + +One gap this leaves is the engine's, not ours, and `settled_failure` pins it: `DeploymentUnit.Delete` +forgives a delete error matching none of the three cases it knows, so a partly deleted set exits +zero with its state dropped. The deployment-history leg is the one path that records and returns it. + ## What each case pins | Case | Claim | @@ -42,6 +61,7 @@ because every adapter call mints its own. | `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | | `batch` | a batch feature with both destinations creates and settles; neither member of its pair can be removed alone, and removing both empties the set, which the update endpoint refuses in favour of delete; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not; a set mixing a batch-path feature with a streaming-path one is refused at create | | `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing; the three members are one cascade group, so each of the three strict subsets of them is refused -- with the message depending on whether the removal names the member a delete is applied to -- and removing all three empties the set, which the update endpoint refuses | +| `settled_failure` | a failure the server hits after reserving an operation comes back as HTTP 200 with the operation FAILED: the adapter recovers the error code from the operation and the per-member outcome from its metadata, the update fails naming each member left behind and writes nothing, and the destroy reports success anyway because the engine forgives a delete error it cannot classify | | `streaming` | a streaming online member needs no schedule and no cron; declaring a schedule state on it, or repointing its destination, plans as a `recreate` rather than an in-place update; with two features materialized to one destination and no offline member between them, each online member is its own cascade group, so one can be removed alone as an `update` | ## Running against a workspace @@ -210,11 +230,10 @@ mixed update to construct. features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case that wants a clean schema has to drop those itself. -- **Partial failure.** One member failing does not stop the others, and the operation settles - FAILED carrying what was updated, deleted and failed in its metadata's `update_progress` or - `deletion_progress`. Reproducing it needs per-member failure injection the fake does not have. - This matters more than it used to: with field updates gone, a partway failure can only be a - partway *deletion*, and nothing already deleted is put back. +- **A real member-teardown failure.** `settled_failure` produces the settled-FAILED shape + from a marker the fake looks for in a feature name, because a real teardown fails on + infrastructure no request can ask for. The adapter's reading of that shape is pinned; that + the server emits it for a genuine failure is not, and cannot be from here. - **A settled-but-failed operation.** Every refusal these cases reach is an HTTP error on the call itself, raised before the operation record exists, so the wait is never what fails. The branch diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl new file mode 100644 index 00000000000..217618ecf63 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl @@ -0,0 +1,79 @@ +bundle: + name: test-matset-settled-$UNIQUE_NAME + +resources: + features: + okfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.ok_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # The fake refuses to tear down any member of a feature whose name carries this marker, which + # is how a fixture asks for a failure that real infrastructure would produce. + failfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.teardown_fails_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: id + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.okfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.okfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.failfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.failfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl b/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl new file mode 100644 index 00000000000..ee6c39a3883 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl @@ -0,0 +1,62 @@ +bundle: + name: test-matset-settled-$UNIQUE_NAME + +resources: + features: + okfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.ok_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # The fake refuses to tear down any member of a feature whose name carries this marker, which + # is how a fixture asks for a failure that real infrastructure would produce. + failfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.teardown_fails_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: id + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.okfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.okfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml b/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt new file mode 100644 index 00000000000..39c0f82f7f1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt @@ -0,0 +1,48 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... +Created features.failfeat +Created features.okfeat +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A member the server cannot tear down settles the update's operation as FAILED and returns +=== it with HTTP 200. The adapter reads the code off the operation and the per-member outcome +=== off its metadata, so the deploy fails naming each member left behind. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID]: Materialization set '[UUID]' was partly updated: 0 materialized features deleted, 2 failed.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) + +Endpoint: n/a +HTTP Status: 200 +API error_code: INTERNAL_ERROR +API message: materialization set [UUID]: Materialization set '[UUID]' was partly updated: 0 materialized features deleted, 2 failed.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. + +Files: 2 uploaded, 0 deleted + +=== Nothing was written, so the removal is still pending +>>> [CLI] bundle plan -o json + +>>> set_action +update + +=== The delete settles FAILED the same way, and here the engine decides what the user sees: +=== its destroy step forgives a delete error matching none of the cases it knows, so the +=== destroy reports success and drops the state while the set stays in the workspace. The leg +=== with deployment-history recording takes the one branch that records and returns the +=== failure, which is why this case runs on the other leg only. +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.failfeat + delete resources.features.okfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/script b/acceptance/bundle/resources/materialization_sets/settled_failure/script new file mode 100644 index 00000000000..b0882e8c2ed --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/script @@ -0,0 +1,36 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-myschema}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +render() { + envsubst < "$1" > databricks.yml +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +render databricks.yml.tmpl +trace $CLI bundle deploy + +title "A member the server cannot tear down settles the update's operation as FAILED and returns" +title "it with HTTP 200. The adapter reads the code off the operation and the per-member outcome" +title "off its metadata, so the deploy fails naming each member left behind." +render drop-failing.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Nothing was written, so the removal is still pending" +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "The delete settles FAILED the same way, and here the engine decides what the user sees:" +title "its destroy step forgives a delete error matching none of the cases it knows, so the" +title "destroy reports success and drops the state while the set stays in the workspace. The leg" +title "with deployment-history recording takes the one branch that records and returns the" +title "failure, which is why this case runs on the other leg only." +trace $CLI bundle destroy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml b/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml new file mode 100644 index 00000000000..61fdc6235a0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml @@ -0,0 +1,12 @@ +# Pinned to the leg without deployment-history recording, where the engine's destroy step forgives +# a delete error it cannot classify. With recording on it takes the branch that records the failure +# and returns it, so the destroy fails instead and the two legs cannot share one golden. +EnvMatrix.DMS = [""] + +# Local only, and not because the claims are redundant: the failure is produced by a marker the +# fake looks for in a feature name, since a real teardown fails on infrastructure no request can +# ask for. Against a workspace this case would simply succeed. +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index daee8567f9b..78a6872ec69 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -2,9 +2,8 @@ package dresources import ( "context" - "errors" "fmt" - "slices" + "net/http" "strconv" "strings" @@ -167,7 +166,7 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma set, err := operation.Wait(ctx) if err != nil { - return "", nil, fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), err) + return "", nil, r.materializationSetOperationFailure(ctx, operation, err) } // The id is what the engine records the resource under, so an operation that settles without // one is a failure even though the wait succeeded. @@ -178,66 +177,60 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma return set.MaterializationSetId, set, nil } -// materializationSetProgressTypes name the error details a partly applied set change carries, one -// per verb that can partly apply. A creation cannot: it has no per-member outcome. -var materializationSetProgressTypes = []string{ - "type.googleapis.com/databricks.featurestore.MaterializationSetUpdateProgress", - "type.googleapis.com/databricks.featurestore.MaterializationSetDeletionProgress", -} +// materializationSetChangeFailures names each member a partly applied change could not reach, from +// whichever progress arm the operation's metadata carries. Empty for a creation, which has no +// per-member outcome, and for a failure raised before the server touched a member. +func materializationSetChangeFailures(metadata *ml.MaterializationSetOperationMetadata) []string { + if metadata == nil { + return nil + } + var members []ml.MaterializedFeatureChangeFailure + switch { + case metadata.UpdateProgress != nil: + members = metadata.UpdateProgress.FailedMaterializedFeatures + case metadata.DeletionProgress != nil: + members = metadata.DeletionProgress.FailedMaterializedFeatures + } -// protoJSONField reads a proto field out of a decoded JSON object under either spelling a -// serializer may emit for it. -func protoJSONField(fields map[string]any, snakeCase, camelCase string) any { - if value, present := fields[snakeCase]; present { - return value + failures := make([]string, 0, len(members)) + for _, member := range members { + failures = append(failures, fmt.Sprintf("%s (%s): %s", + member.MaterializedFeatureId, member.ErrorCode, member.Message)) } - return fields[camelCase] + return failures } -// materializationSetChangeFailures names each member a partly applied change could not reach, -// taken from the progress the server attaches as an error detail. Empty for an error raised before -// the server touched a member, which is every validation failure. -func materializationSetChangeFailures(err error) []string { - apiErr, isAPIError := errors.AsType[*apierr.APIError](err) - if !isAPIError || apiErr == nil { - return nil +// materializationSetOperationFailure restates a settled-but-failed operation as an API error. The +// generated wait renders the failure's code and message into one string and reads no metadata, so +// the error code is recovered structurally and the per-member progress is appended. +// +// StatusCode is the 200 the call actually returned, which maps to no sentinel: a member that could +// not be torn down must not reach the destroy path's permission-denied leniency, whose purpose is +// to forgive a resource that is already gone. +func (r *ResourceMaterializationSet) materializationSetOperationFailure( + ctx context.Context, + operation materializationSetOperation, + waitErr error, +) error { + target := materializationSetOperationTarget(operation) + settled, err := r.client.FeatureEngineering.GetOperation(ctx, ml.GetOperationRequest{ + Name: operation.Name(), + }) + if err != nil || settled.Error == nil { + return fmt.Errorf("%s: %w", target, waitErr) } - var failures []string - for _, detail := range apiErr.ErrorDetails().UnknownDetails { - fields, isObject := detail.(map[string]any) - if !isObject { - continue - } - typeURL, _ := fields["@type"].(string) - if !slices.Contains(materializationSetProgressTypes, typeURL) { - continue - } - entries, _ := protoJSONField( - fields, "failed_materialized_features", "failedMaterializedFeatures").([]any) - for _, entry := range entries { - failure, isObject := entry.(map[string]any) - if !isObject { - continue - } - id, _ := protoJSONField(failure, "materialized_feature_id", "materializedFeatureId").(string) - errorCode, _ := protoJSONField(failure, "error_code", "errorCode").(string) - message, _ := failure["message"].(string) - failures = append(failures, fmt.Sprintf("%s (%s): %s", id, errorCode, message)) + message := settled.Error.Message + if metadata, err := operation.Metadata(); err == nil { + if failures := materializationSetChangeFailures(metadata); len(failures) > 0 { + message += "; members left unchanged: " + strings.Join(failures, "; ") } } - return failures -} - -// withMaterializationSetChangeFailures restates the members a partly applied change left alone in -// the error itself. Applied to the wait as well as the call, so it holds whether the server raises -// the partial failure or settles the operation with it. -func withMaterializationSetChangeFailures(err error) error { - failures := materializationSetChangeFailures(err) - if len(failures) == 0 { - return err + return &apierr.APIError{ + ErrorCode: string(settled.Error.ErrorCode), + Message: target + ": " + message, + StatusCode: http.StatusOK, } - return fmt.Errorf("%w; members left unchanged: %s", err, strings.Join(failures, "; ")) } // materializationSetOperation is what a create, update and delete operation have in common: the @@ -270,13 +263,12 @@ func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, co RequestId: uuid.NewString(), }) if err != nil { - return nil, withMaterializationSetChangeFailures(err) + return nil, err } set, err := operation.Wait(ctx) if err != nil { - return nil, fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), - withMaterializationSetChangeFailures(err)) + return nil, r.materializationSetOperationFailure(ctx, operation, err) } return set, nil } @@ -289,11 +281,10 @@ func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ RequestId: uuid.NewString(), }) if err != nil { - return withMaterializationSetChangeFailures(err) + return err } if err := operation.Wait(ctx); err != nil { - return fmt.Errorf("%s: %w", materializationSetOperationTarget(operation), - withMaterializationSetChangeFailures(err)) + return r.materializationSetOperationFailure(ctx, operation, err) } return nil } diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go index 41aea2e8a8a..cfe46ff17d7 100644 --- a/bundle/direct/dresources/materialization_set_test.go +++ b/bundle/direct/dresources/materialization_set_test.go @@ -1,20 +1,11 @@ package dresources import ( - "bytes" - "context" - "encoding/json" "errors" - "io" - "net/http" - "net/url" "testing" - "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/common" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestMaterializationSetMemberKey(t *testing.T) { @@ -110,68 +101,47 @@ func TestMaterializationSetOperationTarget(t *testing.T) { })) } -// apiErrorWithDetails builds the error the SDK produces from a response body carrying error -// details, which is the only way the per-member outcome of a partly applied change reaches a -// client. -func apiErrorWithDetails(t *testing.T, message string, details []map[string]any) error { - t.Helper() - body, err := json.Marshal(map[string]any{ - "error_code": "RESOURCE_CONFLICT", - "message": message, - "details": details, +func TestMaterializationSetChangeFailuresFromUpdateProgress(t *testing.T) { + failures := materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + UpdateProgress: &ml.MaterializationSetUpdateProgress{ + DeletedMaterializedFeatureIds: []string{"mf-gone"}, + FailedMaterializedFeatures: []ml.MaterializedFeatureChangeFailure{ + { + MaterializedFeatureId: "mf-kept", + ErrorCode: "PERMISSION_DENIED", + Message: "no permission on the online table", + }, + }, + }, }) - require.NoError(t, err) - resp := &http.Response{ - StatusCode: http.StatusConflict, - Status: "409 Conflict", - Request: &http.Request{Method: http.MethodPatch, URL: &url.URL{Path: "/api/2.0/x"}}, - } - return apierr.GetAPIError(context.Background(), common.ResponseWrapper{ - Response: resp, - DebugBytes: body, - ReadCloser: io.NopCloser(bytes.NewReader(body)), - }) + assert.Equal(t, []string{"mf-kept (PERMISSION_DENIED): no permission on the online table"}, failures) } -func updateProgressDetail(failures []map[string]any) map[string]any { - return map[string]any{ - "@type": "type.googleapis.com/databricks.featurestore.MaterializationSetUpdateProgress", - "deleted_materialized_feature_ids": []any{"mf-gone"}, - "failed_materialized_features": failures, - } -} +func TestMaterializationSetChangeFailuresFromDeletionProgress(t *testing.T) { + failures := materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + DeletionProgress: &ml.MaterializationSetDeletionProgress{ + FailedMaterializedFeatures: []ml.MaterializedFeatureChangeFailure{ + {MaterializedFeatureId: "mf-1", ErrorCode: "INTERNAL_ERROR", Message: "pipeline stuck"}, + {MaterializedFeatureId: "mf-2", ErrorCode: "RESOURCE_CONFLICT", Message: "in use"}, + }, + }, + }) -// The server's message defers to an error detail the CLI never prints, so the adapter has to -// restate the failed members in the error itself. -func TestMaterializationSetChangeFailuresAreRestatedInTheError(t *testing.T) { - err := apiErrorWithDetails(t, - "Changed 1 of 2 materialized features in materialization set 'set-1'; see the error "+ - "details for which ones failed.", - []map[string]any{updateProgressDetail([]map[string]any{{ - "materialized_feature_id": "mf-stuck", - "error_code": "RESOURCE_CONFLICT", - "message": "pipeline is updating", - }})}) - - wrapped := withMaterializationSetChangeFailures(err) - assert.Contains(t, wrapped.Error(), "members left unchanged:") - assert.Contains(t, wrapped.Error(), "mf-stuck (RESOURCE_CONFLICT): pipeline is updating") - // The server's own message survives, so nothing a caller already relied on is lost. - assert.Contains(t, wrapped.Error(), "Changed 1 of 2 materialized features") - assert.ErrorIs(t, wrapped, err) + assert.Equal(t, []string{ + "mf-1 (INTERNAL_ERROR): pipeline stuck", + "mf-2 (RESOURCE_CONFLICT): in use", + }, failures) } -// Every validation failure is raised before the server touches a member, so it carries no progress -// and must come back exactly as it arrived. -func TestMaterializationSetChangeFailuresLeavesOtherErrorsAlone(t *testing.T) { - withoutDetails := apiErrorWithDetails(t, "Materialization set 'set-1' does not exist.", nil) - assert.Equal(t, withoutDetails, withMaterializationSetChangeFailures(withoutDetails)) - - // A progress detail naming no failed member is a succeeded change, not a partial one. - empty := apiErrorWithDetails(t, "transient", []map[string]any{updateProgressDetail(nil)}) - assert.Equal(t, empty, withMaterializationSetChangeFailures(empty)) - - plain := errors.New("connection reset") - assert.Equal(t, plain, withMaterializationSetChangeFailures(plain)) +func TestMaterializationSetChangeFailuresWithoutFailedMembers(t *testing.T) { + // A creation carries no progress arm at all, and a change that fully applied carries one + // naming no failure; neither has a member to report. + assert.Empty(t, materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{})) + assert.Empty(t, materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + DeletionProgress: &ml.MaterializationSetDeletionProgress{ + DeletedMaterializedFeatureIds: []string{"mf-1"}, + }, + })) + assert.Empty(t, materializationSetChangeFailures(nil)) } diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 13e02f99672..88d9a858968 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -337,6 +337,96 @@ func (s *FakeWorkspace) materializationSetOperationResponse( return Response{Body: operation} } +// materializedFeatureTeardownFailsMarker is how a fixture asks the fake for a member the backend +// cannot tear down. A real teardown fails on infrastructure -- a job, a pipeline or a table the +// delete cannot reach -- which no request can ask for, so the trigger has to be in the fixture. +const materializedFeatureTeardownFailsMarker = "teardown_fails" + +// materializationSetTeardownFailures reports the members of `targets` the fake refuses to tear +// down, one failure each, in target order. +func materializationSetTeardownFailures( + targets [][]ml.MaterializedFeature, + removedIDs map[string]bool, +) []ml.MaterializedFeatureChangeFailure { + var failures []ml.MaterializedFeatureChangeFailure + for _, target := range targets { + if !strings.Contains(target[0].FeatureName, materializedFeatureTeardownFailsMarker) { + continue + } + for _, member := range target { + if !removedIDs[member.MaterializedFeatureId] { + continue + } + failures = append(failures, ml.MaterializedFeatureChangeFailure{ + MaterializedFeatureId: member.MaterializedFeatureId, + ErrorCode: "INTERNAL_ERROR", + Message: fmt.Sprintf( + "Failed to delete materialized feature '%s'. If the issue persists, please "+ + "retry or contact support.", member.MaterializedFeatureId), + }) + } + } + return failures +} + +// partialMaterializationSetChangeErrorCode collapses the member failures to one code the way the +// backend does: a single code is reported as itself, a permission failure wins over any other, and +// a mix of client errors becomes BAD_REQUEST. +func partialMaterializationSetChangeErrorCode( + failures []ml.MaterializedFeatureChangeFailure, +) string { + codes := map[string]bool{} + for _, failure := range failures { + codes[failure.ErrorCode] = true + } + switch { + case len(codes) == 1: + for code := range codes { + return code + } + return "INTERNAL_ERROR" + case codes["PERMISSION_DENIED"]: + return "PERMISSION_DENIED" + default: + return "INTERNAL_ERROR" + } +} + +// materializationSetFailedOperationResponse settles an operation as FAILED and returns it with +// HTTP 200, which is how the backend reports a change it reserved an operation for and then could +// not finish. The per-member outcome rides on the metadata's progress arm. +func (s *FakeWorkspace) materializationSetFailedOperationResponse( + setID string, + members []ml.MaterializedFeature, + message string, + failures []ml.MaterializedFeatureChangeFailure, + progress func(*ml.MaterializationSetOperationMetadata), +) Response { + settled := ml.MaterializationSetOperationMetadata{ + FeatureFullNames: materializationSetFeatureNames(members), + MaterializationSetId: setID, + State: ml.MaterializationSetOperationMetadataStateFailed, + } + progress(&settled) + + metadata, err := json.Marshal(settled) + if err != nil { + return materializationSetInternalError(err) + } + + operation := ml.Operation{ + Name: "operations/" + nextUUID(), + Done: true, + Metadata: metadata, + Error: &ml.DatabricksServiceExceptionWithDetailsProto{ + ErrorCode: ml.ErrorCode(partialMaterializationSetChangeErrorCode(failures)), + Message: message, + }, + } + s.Operations[operation.Name] = operation + return Response{Body: operation} +} + // materializationSetFeatureNames lists each member's feature once in member order, which is what // the operation metadata reports. func materializationSetFeatureNames(members []ml.MaterializedFeature) []string { @@ -374,6 +464,25 @@ func (s *FakeWorkspace) MaterializationSetsDelete(setID string) Response { if !ok { return materializationSetNotFound(setID) } + + allIDs := map[string]bool{} + for _, member := range existing.MaterializedFeatures { + allIDs[member.MaterializedFeatureId] = true + } + targets := materializationSetDeletionTargets(existing.MaterializedFeatures) + if failures := materializationSetTeardownFailures(targets, allIDs); len(failures) > 0 { + // The set is kept so the members that still need deleting can be retried. + return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, + fmt.Sprintf("Materialization set '%s' was partly deleted: %d materialized features "+ + "deleted, %d failed.", setID, 0, len(failures)), + failures, + func(metadata *ml.MaterializationSetOperationMetadata) { + metadata.DeletionProgress = &ml.MaterializationSetDeletionProgress{ + FailedMaterializedFeatures: failures, + } + }) + } + delete(s.MaterializationSets, setID) return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, struct{}{}) @@ -563,6 +672,21 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res quotedMaterializedFeatureIDs(unrequested))) } + targets := materializationSetDeletionTargets(existing.MaterializedFeatures) + if failures := materializationSetTeardownFailures(targets, removedIDs); len(failures) > 0 { + // Nothing is written: the one target this fake can fail is the only target, so no member + // of the change got through. + return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, + fmt.Sprintf("Materialization set '%s' was partly updated: %d materialized features "+ + "deleted, %d failed.", setID, 0, len(failures)), + failures, + func(metadata *ml.MaterializationSetOperationMetadata) { + metadata.UpdateProgress = &ml.MaterializationSetUpdateProgress{ + FailedMaterializedFeatures: failures, + } + }) + } + existing.MaterializedFeatures = matched s.MaterializationSets[setID] = existing From a5710f6791c6133c24d989ae308522133b5de49d Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Thu, 22 Oct 2026 05:50:04 +0000 Subject: [PATCH 25/47] [FS-2621] Correct the fake from what the live promotion tip returns Three messages had been reworded, and two create-path refusals had moved from an HTTP 400 to a settled FAILED operation: the batch-and-streaming mixing rule and the sawtooth triple rule both live inside the batch create the handler calls after reserving its operation record, as do the two destination rules. Only an empty member list, a colliding set id and the wire-level schedule-state hook are reached early enough to raise. The fake now splits its create rules around the reservation and returns a refusal value the raise and settle paths each report their own way. Four goldens move with it, every one of them corrected from what the live run returned rather than from reading the handler. All six Cloud = true cases pass against universe cd7a68312ca3, run one at a time. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 17 ++- .../materialization_sets/batch/output.txt | 15 +-- .../materialization_sets/batch/script | 3 +- .../membership/output.txt | 4 +- .../materialization_sets/sawtooth/output.txt | 12 +- .../settled_failure/output.txt | 4 +- libs/testserver/materialization_set.go | 107 +++++++++++------- 7 files changed, 101 insertions(+), 61 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 23b4fe25f1f..b473ca72a72 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -31,8 +31,14 @@ because every adapter call mints its own. ## A failure after the operation is reserved is not an HTTP error -The backend raises only while it can still reject the request outright -- every validation, which -runs before it reserves an operation record. Past that point a failure is the operation's outcome, +The backend raises only while it can still reject the request outright, and that boundary sits +earlier than "every validation". On the create path only three rules are reached before the record +is reserved: a member list that is empty, a supplied set id that already exists, and the schedule +states agreeing, which a wire-level validation hook checks before the handler runs at all. The +batch-and-streaming mixing rule, both destination rules and the sawtooth triple rule all live inside +the batch create the handler calls *after* reserving, so each of them settles the record and comes +back as HTTP 200. Measured against the live build, not read off the handler: `batch` and `sawtooth` +each failed on a golden still expecting a 400. Past that point a failure is the operation's outcome, so all three verbs settle the record FAILED and return it with HTTP 200, carrying the error code and message on the operation and the per-member outcome on the metadata's progress arm. A creation has no progress arm: it has no per-member outcome to report. @@ -200,6 +206,13 @@ the batch-versus-streaming mixing refusal, and the `streaming` case's claim that materialized to one destination can share a set, which until then rested on the shared-destination rule comparing a `(offline, Option[online])` pair that is identical for both. +**Fourth build, universe `cd7a68312ca3`** -- the promotion tip, carrying the settled-failure +contract. All six `Cloud = true` cases pass against it, run one at a time. Three messages had been +reworded (`materializationSetMembersCannotBeRemovedAlone`, and both partial-change messages, which +now count what got through and point at the operation's progress), and two refusals moved from an +HTTP 400 to a settled FAILED operation. The fake and four goldens were corrected from what the live +run returned. + ## Verified against a workspace, with no case of their own Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index 9aad1a0ef30..b56ee70c017 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -49,12 +49,12 @@ Files: 2 uploaded, 0 deleted >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. Files: 2 uploaded, 0 deleted @@ -145,15 +145,16 @@ Files: 2 uploaded, 0 deleted === A set cannot mix a feature that materializes on the batch path with one that materializes === on the streaming path. The refusal above left no set behind, so this reaches the create -=== endpoint, and it is raised before the shared-destination rule the two features also break. +=== endpoint -- which reserves its operation record before it reaches this rule, so the refusal +=== settles that record and arrives as HTTP 200 rather than as an error on the call. >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Created features.streamfeat -Error: cannot create resources.materialization_sets.set1: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (400 INVALID_PARAMETER_VALUE) +Error: cannot create resources.materialization_sets.set1: materialization set [UUID]: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (200 INVALID_PARAMETER_VALUE) -Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] -HTTP Status: 400 Bad Request +Endpoint: n/a +HTTP Status: 200 API error_code: INVALID_PARAMETER_VALUE -API message: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] +API message: materialization set [UUID]: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script index f7e135e497c..cdbb6b2ea7f 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/script +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -76,7 +76,8 @@ trace musterr $CLI bundle deploy --auto-approve title "A set cannot mix a feature that materializes on the batch path with one that materializes" title "on the streaming path. The refusal above left no set behind, so this reaches the create" -title "endpoint, and it is raised before the shared-destination rule the two features also break." +title "endpoint -- which reserves its operation record before it reaches this rule, so the refusal" +title "settles that record and arrives as HTTP 200 rather than as an error on the call." render mixed.yml.tmpl trace musterr $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index c7b7620a49f..35a4b1df921 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -16,12 +16,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === refused, because the delete also reaches the offline member the config still declares >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request doesn't supply. Remove '[UUID]' as well. +API message: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt index e5ac092c9ea..21179031cbe 100644 --- a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -6,12 +6,12 @@ >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... Created features.sawfeat -Error: cannot create resources.materialization_sets.set1: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (400 BAD_REQUEST) +Error: cannot create resources.materialization_sets.set1: materialization set [UUID]: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (200 BAD_REQUEST) -Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] -HTTP Status: 400 Bad Request +Endpoint: n/a +HTTP Status: 200 API error_code: BAD_REQUEST -API message: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. +API message: materialization set [UUID]: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. Files: 7 uploaded, 0 deleted @@ -66,12 +66,12 @@ update >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request doesn't supply. Remove '[UUID]', '[UUID]' as well. (400 INVALID_PARAMETER_VALUE) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request doesn't supply. Remove '[UUID]', '[UUID]' as well. +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request does not supply. Remove those as well. Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt index 39c0f82f7f1..c825a32f3fb 100644 --- a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt @@ -17,12 +17,12 @@ update >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID]: Materialization set '[UUID]' was partly updated: 0 materialized features deleted, 2 failed.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID]: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) Endpoint: n/a HTTP Status: 200 API error_code: INTERNAL_ERROR -API message: materialization set [UUID]: Materialization set '[UUID]' was partly updated: 0 materialized features deleted, 2 failed.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. +API message: materialization set [UUID]: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. Files: 2 uploaded, 0 deleted diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 88d9a858968..82f3dd8fd32 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -15,6 +15,14 @@ import ( // backend's own message text, so a golden recorded locally is a claim about the real API that the // cloud run then checks. +// materializationSetRefusal is a rule's verdict, separated from how it is reported: a rule the +// backend reaches before reserving an operation record becomes an HTTP error, and one it reaches +// after becomes a settled FAILED operation. +type materializationSetRefusal struct { + errorCode string + message string +} + func materializationSetError(statusCode int, errorCode, message string) Response { return Response{ StatusCode: statusCode, @@ -123,7 +131,7 @@ func featureMaterializationDestination(members []ml.MaterializedFeature) string // validateSharedMaterializationDestination mirrors the create-path rule that every feature in a // set asks for an identical destination pair. Because the pair carries the online config, a set // cannot mix a feature materialized to both stores with one materialized offline only. -func validateSharedMaterializationDestination(members []ml.MaterializedFeature) *Response { +func validateSharedMaterializationDestination(members []ml.MaterializedFeature) *materializationSetRefusal { membersByFeature := map[string][]ml.MaterializedFeature{} featureOrder := []string{} for _, member := range members { @@ -138,16 +146,17 @@ func validateSharedMaterializationDestination(members []ml.MaterializedFeature) destinations[featureMaterializationDestination(membersByFeature[featureName])] = true } if len(destinations) > 1 { - response := materializationSetError(http.StatusBadRequest, "BAD_REQUEST", - "All features must share the same materialization destination.") - return &response + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: "All features must share the same materialization destination.", + } } return validateOfflineAndOnlineDestinationsDiffer(members) } // validateOfflineAndOnlineDestinationsDiffer mirrors the create-path rule that a feature // materialized to both stores must separate them by catalog, schema or prefix. -func validateOfflineAndOnlineDestinationsDiffer(members []ml.MaterializedFeature) *Response { +func validateOfflineAndOnlineDestinationsDiffer(members []ml.MaterializedFeature) *materializationSetRefusal { var offline *ml.OfflineStoreConfig var online *ml.OnlineStoreConfig for _, member := range members { @@ -165,11 +174,12 @@ func validateOfflineAndOnlineDestinationsDiffer(members []ml.MaterializedFeature online.TableNamePrefix != offline.TableNamePrefix { return nil } - response := materializationSetError(http.StatusBadRequest, "BAD_REQUEST", - fmt.Sprintf("Offline and online materialization destinations must differ. Found "+ + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf("Offline and online materialization destinations must differ. Found "+ "identical catalog names '%s', schema names '%s', and table name prefixes '%s'.", - offline.CatalogName, offline.SchemaName, offline.TableNamePrefix)) - return &response + offline.CatalogName, offline.SchemaName, offline.TableNamePrefix), + } } // sawtoothMaterializedFeatureCount is how many materialized features a sawtooth window produces: @@ -194,7 +204,7 @@ func (s *FakeWorkspace) isSawtoothFeature(featureName string) bool { // trigger. An unknown feature falls in neither group, for the reason isSawtoothFeature gives. func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( members []ml.MaterializedFeature, -) *Response { +) *materializationSetRefusal { var batchFeatureNames, streamingFeatureNames []string for _, featureName := range materializationSetFeatureNames(members) { feature, stored := s.Features[featureName] @@ -218,13 +228,14 @@ func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( return nil } - response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", - fmt.Sprintf("Batch and streaming features cannot be mixed in a single "+ + return &materializationSetRefusal{ + errorCode: "INVALID_PARAMETER_VALUE", + message: fmt.Sprintf("Batch and streaming features cannot be mixed in a single "+ "materialize_features call because they require different trigger types. Issue two "+ "separate calls: one for batch features and one for streaming features. Batch "+ "features: %s. Streaming features: %s", - strings.Join(batchFeatureNames, ", "), strings.Join(streamingFeatureNames, ", "))) - return &response + strings.Join(batchFeatureNames, ", "), strings.Join(streamingFeatureNames, ", ")), + } } // validateSawtoothMaterializedFeatures mirrors the rule that a sawtooth feature declares all three @@ -232,7 +243,9 @@ func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( // stamps a set id on every member before the backend reaches its normalization, and that // normalization is skipped whenever the id is set, so a member maps to exactly one materialized // feature and the set stays diffable. -func (s *FakeWorkspace) validateSawtoothMaterializedFeatures(members []ml.MaterializedFeature) *Response { +func (s *FakeWorkspace) validateSawtoothMaterializedFeatures( + members []ml.MaterializedFeature, +) *materializationSetRefusal { byFeature := map[string][]ml.MaterializedFeature{} var featureOrder []string for _, member := range members { @@ -259,11 +272,14 @@ func (s *FakeWorkspace) validateSawtoothMaterializedFeatures(members []ml.Materi } if len(byFeature[featureName]) != sawtoothMaterializedFeatureCount || offlineBatch != 1 || onlineBatch != 1 || onlineStreaming != 1 { - response := materializationSetError(http.StatusBadRequest, "BAD_REQUEST", fmt.Sprintf( - "Feature '%s' uses a sawtooth window, which requires an offline batch "+ - "materialization, an online batch materialization and an online streaming "+ - "materialization. Declare one materialize feature request for each.", featureName)) - return &response + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf( + "Feature '%s' uses a sawtooth window, which requires an offline batch "+ + "materialization, an online batch materialization and an online streaming "+ + "materialization. Declare one materialize feature request for each.", + featureName), + } } } return nil @@ -285,17 +301,22 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { return *bad } - if bad := s.validateBatchAndStreamingFeaturesNotMixed(set.MaterializedFeatures); bad != nil { - return *bad - } - if bad := validateSharedMaterializationDestination(set.MaterializedFeatures); bad != nil { - return *bad - } - if bad := s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures); bad != nil { - return *bad - } - set.MaterializationSetId = nextUUID() + + // The operation record is reserved here, so what follows is the operation's outcome rather + // than a rejected request: the backend reaches these rules inside the batch create it calls + // after reserving, and reports them by settling the record. + for _, bad := range []*materializationSetRefusal{ + s.validateBatchAndStreamingFeaturesNotMixed(set.MaterializedFeatures), + validateSharedMaterializationDestination(set.MaterializedFeatures), + s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures), + } { + if bad == nil { + continue + } + return s.materializationSetFailedOperationResponse(set.MaterializationSetId, + set.MaterializedFeatures, bad.errorCode, bad.message, nil) + } for i, member := range set.MaterializedFeatures { set.MaterializedFeatures[i] = populateMaterializationSetMember(member, set.MaterializationSetId) } @@ -398,8 +419,8 @@ func partialMaterializationSetChangeErrorCode( func (s *FakeWorkspace) materializationSetFailedOperationResponse( setID string, members []ml.MaterializedFeature, + errorCode string, message string, - failures []ml.MaterializedFeatureChangeFailure, progress func(*ml.MaterializationSetOperationMetadata), ) Response { settled := ml.MaterializationSetOperationMetadata{ @@ -407,7 +428,9 @@ func (s *FakeWorkspace) materializationSetFailedOperationResponse( MaterializationSetId: setID, State: ml.MaterializationSetOperationMetadataStateFailed, } - progress(&settled) + if progress != nil { + progress(&settled) + } metadata, err := json.Marshal(settled) if err != nil { @@ -419,7 +442,7 @@ func (s *FakeWorkspace) materializationSetFailedOperationResponse( Done: true, Metadata: metadata, Error: &ml.DatabricksServiceExceptionWithDetailsProto{ - ErrorCode: ml.ErrorCode(partialMaterializationSetChangeErrorCode(failures)), + ErrorCode: ml.ErrorCode(errorCode), Message: message, }, } @@ -473,9 +496,10 @@ func (s *FakeWorkspace) MaterializationSetsDelete(setID string) Response { if failures := materializationSetTeardownFailures(targets, allIDs); len(failures) > 0 { // The set is kept so the members that still need deleting can be retried. return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, - fmt.Sprintf("Materialization set '%s' was partly deleted: %d materialized features "+ - "deleted, %d failed.", setID, 0, len(failures)), - failures, + partialMaterializationSetChangeErrorCode(failures), + fmt.Sprintf("Deleted %d of %d materialized features in materialization set '%s'. "+ + "The operation's progress lists which deletion succeeded, which failed and why.", + 0, len(failures), setID), func(metadata *ml.MaterializationSetOperationMetadata) { metadata.DeletionProgress = &ml.MaterializationSetDeletionProgress{ FailedMaterializedFeatures: failures, @@ -666,9 +690,9 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res } return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", fmt.Sprintf("Materialized features %s of feature '%s' are deleted only by deleting "+ - "materialized features %s, which the request doesn't supply. Remove %s as well.", + "materialized features %s, which the request does not supply. Remove those as "+ + "well.", quotedMaterializedFeatureIDs(removedForFeature), featureName, - quotedMaterializedFeatureIDs(unrequested), quotedMaterializedFeatureIDs(unrequested))) } @@ -677,9 +701,10 @@ func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Res // Nothing is written: the one target this fake can fail is the only target, so no member // of the change got through. return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, - fmt.Sprintf("Materialization set '%s' was partly updated: %d materialized features "+ - "deleted, %d failed.", setID, 0, len(failures)), - failures, + partialMaterializationSetChangeErrorCode(failures), + fmt.Sprintf("Changed %d of %d materialized features in materialization set '%s'. "+ + "The operation's progress lists which update succeeded, which failed and why.", + 0, len(failures), setID), func(metadata *ml.MaterializationSetOperationMetadata) { metadata.UpdateProgress = &ml.MaterializationSetUpdateProgress{ FailedMaterializedFeatures: failures, From d5b235bd8f42e3d82afe13149f2458076d55497f Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Thu, 22 Oct 2026 06:12:43 +0000 Subject: [PATCH 26/47] [FS-2621] Name the outcome when an operation settles FAILED The summary line ends in the 200 the call returned, which read as a contradiction beside an error code on its own. Saying the operation failed makes the status mean what it is: the call succeeded, the operation it returned did not. Co-authored-by: Isaac --- .../bundle/resources/materialization_sets/batch/output.txt | 4 ++-- .../bundle/resources/materialization_sets/sawtooth/output.txt | 4 ++-- .../resources/materialization_sets/settled_failure/output.txt | 4 ++-- bundle/direct/dresources/materialization_set.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index b56ee70c017..2d148021ce4 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -150,11 +150,11 @@ Files: 2 uploaded, 0 deleted >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Created features.streamfeat -Error: cannot create resources.materialization_sets.set1: materialization set [UUID]: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (200 INVALID_PARAMETER_VALUE) +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (200 INVALID_PARAMETER_VALUE) Endpoint: n/a HTTP Status: 200 API error_code: INVALID_PARAMETER_VALUE -API message: materialization set [UUID]: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] +API message: materialization set [UUID] operation failed: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt index 21179031cbe..1cb3dd4ed82 100644 --- a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -6,12 +6,12 @@ >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... Created features.sawfeat -Error: cannot create resources.materialization_sets.set1: materialization set [UUID]: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (200 BAD_REQUEST) +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (200 BAD_REQUEST) Endpoint: n/a HTTP Status: 200 API error_code: BAD_REQUEST -API message: materialization set [UUID]: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. +API message: materialization set [UUID] operation failed: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. Files: 7 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt index c825a32f3fb..38912722d51 100644 --- a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt @@ -17,12 +17,12 @@ update >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... -Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID]: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID] operation failed: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) Endpoint: n/a HTTP Status: 200 API error_code: INTERNAL_ERROR -API message: materialization set [UUID]: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. +API message: materialization set [UUID] operation failed: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. Files: 2 uploaded, 0 deleted diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 78a6872ec69..6733ad67d32 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -228,7 +228,7 @@ func (r *ResourceMaterializationSet) materializationSetOperationFailure( } return &apierr.APIError{ ErrorCode: string(settled.Error.ErrorCode), - Message: target + ": " + message, + Message: target + " operation failed: " + message, StatusCode: http.StatusOK, } } From 80ac9ba1766d6116da1f344cb2a502cca2f00bc1 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 01:10:02 +0000 Subject: [PATCH 27/47] [FS-2621] Declare one feature's materializations in a shorthand the CLI fans out A sawtooth-window feature needs three members and a feature materialized to both stores needs two, so the member list has exactly one legal shape per feature and the author repeats the feature name and the destinations to reach it. Two authoring fields now declare one feature's materialization once, and a mutator fans each entry out into the members it is made of before the plan. The mutator judges nothing: it moves named fields across and lets the service refuse a malformation, which it already does by name. Co-authored-by: Isaac --- .../shorthand/batch-pair.yml.tmpl | 77 ++++++ .../shorthand/databricks.yml.tmpl | 41 ++++ .../shorthand/manual-schedule.yml.tmpl | 42 ++++ .../shorthand/moved.yml.tmpl | 41 ++++ .../shorthand/out.test.toml | 2 + .../materialization_sets/shorthand/output.txt | 130 ++++++++++ .../materialization_sets/shorthand/script | 79 ++++++ .../materialization_sets/shorthand/test.toml | 30 +++ .../expand_materialization_set_members.go | 118 +++++++++ ...terialization_set_members_coverage_test.go | 116 +++++++++ ...expand_materialization_set_members_test.go | 229 ++++++++++++++++++ .../resourcemutator/resource_mutator.go | 6 + .../config/resources/materialization_set.go | 13 + libs/testserver/materialization_set.go | 22 ++ 14 files changed, 946 insertions(+) create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/output.txt create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/script create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/test.toml create mode 100644 bundle/config/mutator/resourcemutator/expand_materialization_set_members.go create mode 100644 bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go create mode 100644 bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl new file mode 100644 index 00000000000..1fb2d6fface --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl @@ -0,0 +1,77 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + pairfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.pair_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + otherfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.other_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # pairfeat is declared once and expanded into its offline and online members; otherfeat is + # written out in full. Every feature in a set must ask for the same destination pair, so the + # two declarations name identical destinations. + batch_offline_and_online_materialized_features: + - feature_name: ${resources.features.pairfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + materialized_features: + - feature_name: ${resources.features.otherfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.otherfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl new file mode 100644 index 00000000000..8e97eb27faf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.shd_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + # One entry for the whole of this feature's materialization. Both destinations and both + # triggers are written once; the CLI splits them across the three members the service wants. + sawtooth_materialized_features: + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl new file mode 100644 index 00000000000..438249cca84 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.shd_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + # One entry for the whole of this feature's materialization. Both destinations and both + # triggers are written once; the CLI splits them across the three members the service wants. + sawtooth_materialized_features: + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl new file mode 100644 index 00000000000..0ff83c0eb16 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.shd_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + # One entry for the whole of this feature's materialization. Both destinations and both + # triggers are written once; the CLI splits them across the three members the service wants. + sawtooth_materialized_features: + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_moved + cron_schedule_trigger: + mode: DERIVED + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml b/acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/output.txt b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt new file mode 100644 index 00000000000..f449c3a184c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt @@ -0,0 +1,130 @@ + +=== The shorthand judges nothing it is given. A batch schedule the service refuses is reported +=== by the service, naming the feature, rather than rejected while the config is being read. +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.sawfeat +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Feature 'danielb.fs2680.shd_[UNIQUE_NAME]' must declare a DERIVED cron schedule on its batch materialize feature requests. The sawtooth batch materializations refresh on a daily schedule the service computes, so a caller neither pins nor omits it. (200 BAD_REQUEST) + +Endpoint: n/a +HTTP Status: 200 +API error_code: BAD_REQUEST +API message: materialization set [UUID] operation failed: Feature 'danielb.fs2680.shd_[UNIQUE_NAME]' must declare a DERIVED cron schedule on its batch materialize feature requests. The sawtooth batch materializations refresh on a daily schedule the service computes, so a caller neither pins nor omits it. + +Files: 5 uploaded, 0 deleted + +=== Declaring the same feature once with a DERIVED batch schedule creates the three members it +=== is made of, and neither shorthand field survives into the summary +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +>>> members +{ + "sawtooth_shorthand_remains": false, + "batch_pair_shorthand_remains": false, + "members": [ + { + "feature": "sawfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "sawfeat", + "destination": "online", + "trigger": "batch" + }, + { + "feature": "sawfeat", + "destination": "online", + "trigger": "streaming" + } + ] +} + +=== A redeploy plans nothing, so the expansion is the same on every invocation +>>> [CLI] bundle plan -o json + +>>> set_action +skip + +=== The online destination is written once, so one edit reaches both online members. The offline +=== member, whose destination did not change, is absent from the changes. +>>> [CLI] bundle plan -o json + +>>> set_action +recreate + +>>> recreated_fields +materialized_features[member='danielb.fs2680.shd_[UNIQUE_NAME]|danielb|false'].online_store_config.table_name_prefix +materialized_features[member='danielb.fs2680.shd_[UNIQUE_NAME]|danielb|true'].online_store_config.table_name_prefix + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +skip + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.sawfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default + +Destroy: 2 deleted + +=== A set may mix a feature declared in shorthand with members written out in full: the +=== expansion is appended to what the config already lists +>>> create_table.py danielb.fs2680.src_[UNIQUE_NAME] +Created table danielb.fs2680.src_[UNIQUE_NAME] +Inserted sample data into danielb.fs2680.src_[UNIQUE_NAME] +Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.otherfeat +Created features.pairfeat +Created materialization_sets.set1 +Files: 6 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> members +{ + "sawtooth_shorthand_remains": false, + "batch_pair_shorthand_remains": false, + "members": [ + { + "feature": "otherfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "otherfeat", + "destination": "online", + "trigger": "batch" + }, + { + "feature": "pairfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "pairfeat", + "destination": "online", + "trigger": "batch" + } + ] +} diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/script b/acceptance/bundle/resources/materialization_sets/shorthand/script new file mode 100644 index 00000000000..41c0e390756 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/script @@ -0,0 +1,79 @@ +# The sawtooth half needs a Stream source, and a Stream is not a bundle resource, so it references +# one that already exists. The batch half creates its own Delta source. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +# The shorthand fields must not survive the expansion, so the summary is the whole picture of what +# was declared. +members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | { + sawtooth_shorthand_remains: has("sawtooth_materialized_features"), + batch_pair_shorthand_remains: has("batch_offline_and_online_materialized_features"), + members: [.materialized_features[] | { + feature: (.feature_name | ltrimstr("${resources.features.") | rtrimstr(".full_name}")), + destination: (if .online_store_config != null then "online" else "offline" end), + trigger: (if .streaming_mode != null then "streaming" else "batch" end) + }] + }' +} + +# Only the fields that force the recreate, so the claim is which members the one edit reached. +recreated_fields() { + jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries + | map(select(.value.action == "recreate")) | map(.key)[]' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +title "The shorthand judges nothing it is given. A batch schedule the service refuses is reported" +title "by the service, naming the feature, rather than rejected while the config is being read." +render manual-schedule.yml.tmpl +trace musterr $CLI bundle deploy + +title "Declaring the same feature once with a DERIVED batch schedule creates the three members it" +title "is made of, and neither shorthand field survives into the summary" +render databricks.yml.tmpl +trace $CLI bundle deploy +trace members + +title "A redeploy plans nothing, so the expansion is the same on every invocation" +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "The online destination is written once, so one edit reaches both online members. The offline" +title "member, whose destination did not change, is absent from the changes." +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace recreated_fields +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +trace $CLI bundle destroy --auto-approve + +title "A set may mix a feature declared in shorthand with members written out in full: the" +title "expansion is appended to what the config already lists" +trace create_table.py "$TABLE_NAME" +render batch-pair.yml.tmpl +trace $CLI bundle deploy +trace members + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/test.toml b/acceptance/bundle/resources/materialization_sets/shorthand/test.toml new file mode 100644 index 00000000000..05d46f50e20 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/test.toml @@ -0,0 +1,30 @@ +# Mirrors sawtooth/test.toml: pinned to a workspace holding a Stream, since a Stream is not a +# bundle resource. Override MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM +# to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# The batch half creates and drops a Delta source, so the local run needs the same two stubs +# batch/test.toml carries. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_members.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_members.go new file mode 100644 index 00000000000..013b0745c29 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_members.go @@ -0,0 +1,118 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type expandMaterializationSetMembers struct{} + +func ExpandMaterializationSetMembers() bundle.Mutator { + return &expandMaterializationSetMembers{} +} + +func (m *expandMaterializationSetMembers) Name() string { + return "ExpandMaterializationSetMembers" +} + +// sawtoothMembers builds the three materialized features a sawtooth-window feature is made of. The +// destination and the trigger are split across them; every other field the entry carries is held by +// all three. +// +// materialized_feature_id is deliberately absent: the server assigns one per member, so a single +// declared id cannot belong to three. ForceSendFields is absent because the dyn-to-typed +// conversion rebuilds it per member from the fields that member holds. +func sawtoothMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { + return []ml.MaterializedFeature{ + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + }, + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OnlineStoreConfig: entry.OnlineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + }, + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OnlineStoreConfig: entry.OnlineStoreConfig, + StreamingMode: entry.StreamingMode, + }, + } +} + +// batchOfflineAndOnlineMembers builds the two materialized features a feature materialized to both +// stores is made of. Only the destination is split; every other field the entry carries, trigger +// included, is held by both. +// +// The omissions are the same as in sawtoothMembers. +func batchOfflineAndOnlineMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { + return []ml.MaterializedFeature{ + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + StreamingMode: entry.StreamingMode, + }, + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OnlineStoreConfig: entry.OnlineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + StreamingMode: entry.StreamingMode, + }, + } +} + +func (m *expandMaterializationSetMembers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, set := range b.Config.Resources.MaterializationSets { + if set == nil { + continue + } + + // Appended rather than assigned: a set may declare one feature in shorthand and another + // as members, and the adapter keys members by content rather than by position. + for _, entry := range set.SawtoothMaterializedFeatures { + set.MaterializedFeatures = append(set.MaterializedFeatures, sawtoothMembers(entry)...) + } + for _, entry := range set.BatchOfflineAndOnlineMaterializedFeatures { + set.MaterializedFeatures = append(set.MaterializedFeatures, batchOfflineAndOnlineMembers(entry)...) + } + + set.SawtoothMaterializedFeatures = nil + set.BatchOfflineAndOnlineMaterializedFeatures = nil + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go new file mode 100644 index 00000000000..9ec54471ffa --- /dev/null +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go @@ -0,0 +1,116 @@ +package resourcemutator_test + +import ( + "reflect" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// memberFieldsNotCarried are the member fields a shorthand entry never contributes to the members +// it expands into. All but the first are OUTPUT_ONLY in the spec, so the schema generator already +// keeps them out of the bundle schema and no config can set them. +// +// materialized_feature_id is the one field a shorthand entry can set and have ignored: the server +// assigns one per member, so a single declared id cannot belong to all of them. +var memberFieldsNotCarried = map[string]bool{ + "MaterializedFeatureId": true, + "MaterializationSetId": true, + "IsOnline": true, + "JobId": true, + "PipelineId": true, + "TableName": true, + "LastMaterializationTime": true, + "LatestBackfillOperation": true, +} + +// memberFieldsDerivedByConfigLayer are neither declared nor carried: the dyn-to-typed conversion +// rebuilds them per member from the fields that member holds. ForceSendFields comes back naming +// each expanded member's own pointer fields, so an explicitly set zero value survives the +// expansion without the mutator naming it. +var memberFieldsDerivedByConfigLayer = map[string]bool{ + "ForceSendFields": true, +} + +// memberWithEveryFieldSet returns a materialized feature whose every field holds a non-zero value, +// so that a field missing from every expanded member is visible as a zero. +func memberWithEveryFieldSet(t *testing.T) ml.MaterializedFeature { + t.Helper() + + var member ml.MaterializedFeature + value := reflect.ValueOf(&member).Elem() + for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + target := value.FieldByName(field.Name) + switch field.Type.Kind() { + case reflect.String: + target.SetString("x") + case reflect.Bool: + target.SetBool(true) + case reflect.Int, reflect.Int32, reflect.Int64: + target.SetInt(1) + case reflect.Pointer: + target.Set(reflect.New(field.Type.Elem())) + case reflect.Map: + entry := reflect.MakeMap(field.Type) + entry.SetMapIndex(reflect.ValueOf("k"), reflect.ValueOf("v")) + target.Set(entry) + case reflect.Slice: + target.Set(reflect.MakeSlice(field.Type, 1, 1)) + default: + require.Failf(t, "unhandled field kind", + "field %s has kind %s, which this test does not know how to populate", + field.Name, field.Type.Kind()) + } + } + return member +} + +// TestExpandMaterializationSetMembersCoversEveryMemberField keeps each expansion's field list +// exhaustive. Members are built by naming their fields rather than by copying the entry, so a +// member field added to the proto later reaches no expanded member and nothing else notices. +func TestExpandMaterializationSetMembersCoversEveryMemberField(t *testing.T) { + entry := memberWithEveryFieldSet(t) + + for name, set := range map[string]*resources.MaterializationSet{ + "sawtooth_materialized_features": { + SawtoothMaterializedFeatures: []ml.MaterializedFeature{entry}, + }, + "batch_offline_and_online_materialized_features": { + BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{entry}, + }, + } { + t.Run(name, func(t *testing.T) { + expanded := expandSet(t, set) + require.NotEmpty(t, expanded.MaterializedFeatures) + + for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + if memberFieldsDerivedByConfigLayer[field.Name] { + continue + } + + carried := false + for _, member := range expanded.MaterializedFeatures { + if !reflect.ValueOf(member).FieldByName(field.Name).IsZero() { + carried = true + break + } + } + + if memberFieldsNotCarried[field.Name] { + assert.Falsef(t, carried, + "field %s is listed in memberFieldsNotCarried but reached an expanded "+ + "member; drop it from that list or stop carrying it", field.Name) + continue + } + assert.Truef(t, carried, + "field %s reached no member of the %s expansion. Add it to the member "+ + "literals in expand_materialization_set_members.go, or to "+ + "memberFieldsNotCarried with the reason it is server-owned", + field.Name, name) + } + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go new file mode 100644 index 00000000000..df46c667616 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go @@ -0,0 +1,229 @@ +package resourcemutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + shorthandFeatureName = "main.myschema.feat" + shorthandCronExpr = "0 0 * * * ?" +) + +func shorthandOfflineStore() *ml.OfflineStoreConfig { + return &ml.OfflineStoreConfig{ + CatalogName: "main", + SchemaName: "myschema", + TableNamePrefix: "feat_off", + } +} + +func shorthandOnlineStore() *ml.OnlineStoreConfig { + return &ml.OnlineStoreConfig{ + OnlineStoreName: "store", + CatalogName: "main", + SchemaName: "myschema", + TableNamePrefix: "feat", + } +} + +func expandSet(t *testing.T, set *resources.MaterializationSet) *resources.MaterializationSet { + t.Helper() + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + MaterializationSets: map[string]*resources.MaterializationSet{"set1": set}, + }, + }, + } + diags := bundle.Apply(t.Context(), b, resourcemutator.ExpandMaterializationSetMembers()) + require.NoError(t, diags.Error()) + + return b.Config.Resources.MaterializationSets["set1"] +} + +func TestExpandMaterializationSetMembersSawtooth(t *testing.T) { + set := expandSet(t, &resources.MaterializationSet{ + MaterializationSet: ml.MaterializationSet{}, + SawtoothMaterializedFeatures: []ml.MaterializedFeature{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 3) + + offlineBatch := set.MaterializedFeatures[0] + assert.Equal(t, shorthandOfflineStore(), offlineBatch.OfflineStoreConfig) + assert.Nil(t, offlineBatch.OnlineStoreConfig) + assert.Equal(t, ml.CronScheduleModeDerived, offlineBatch.CronScheduleTrigger.Mode) + assert.Nil(t, offlineBatch.StreamingMode) + + onlineBatch := set.MaterializedFeatures[1] + assert.Nil(t, onlineBatch.OfflineStoreConfig) + assert.Equal(t, shorthandOnlineStore(), onlineBatch.OnlineStoreConfig) + assert.Equal(t, ml.CronScheduleModeDerived, onlineBatch.CronScheduleTrigger.Mode) + assert.Nil(t, onlineBatch.StreamingMode) + + onlineStreaming := set.MaterializedFeatures[2] + assert.Nil(t, onlineStreaming.OfflineStoreConfig) + assert.Equal(t, shorthandOnlineStore(), onlineStreaming.OnlineStoreConfig) + assert.Nil(t, onlineStreaming.CronScheduleTrigger) + assert.Equal(t, ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm, onlineStreaming.StreamingMode.Mode) + + for _, member := range set.MaterializedFeatures { + assert.Equal(t, shorthandFeatureName, member.FeatureName) + } + assert.Nil(t, set.SawtoothMaterializedFeatures) +} + +func TestExpandMaterializationSetMembersBatchOfflineAndOnline(t *testing.T) { + trigger := &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr} + set := expandSet(t, &resources.MaterializationSet{ + BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: trigger, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 2) + + offline := set.MaterializedFeatures[0] + assert.Equal(t, shorthandOfflineStore(), offline.OfflineStoreConfig) + assert.Nil(t, offline.OnlineStoreConfig) + + online := set.MaterializedFeatures[1] + assert.Nil(t, online.OfflineStoreConfig) + assert.Equal(t, shorthandOnlineStore(), online.OnlineStoreConfig) + + // The trigger is not split for a batch pair: both members refresh together. + assert.Equal(t, trigger, offline.CronScheduleTrigger) + assert.Equal(t, trigger, online.CronScheduleTrigger) + assert.Nil(t, set.BatchOfflineAndOnlineMaterializedFeatures) +} + +func TestExpandMaterializationSetMembersCarriesSharedFields(t *testing.T) { + notifications := &ml.MaterializationNotifications{ + OnFailure: &ml.MaterializationFailureNotification{EmailAddresses: []string{"a@example.com"}}, + } + entry := ml.MaterializedFeature{ + FeatureName: shorthandFeatureName, + Tags: map[string]string{"team": "feature-store"}, + BudgetPolicyId: "budget-1", + Notifications: notifications, + PipelineScheduleState: ml.MaterializedFeaturePipelineScheduleStatePaused, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + } + + for name, set := range map[string]*resources.MaterializationSet{ + "sawtooth": {SawtoothMaterializedFeatures: []ml.MaterializedFeature{entry}}, + "batch pair": {BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{entry}}, + } { + t.Run(name, func(t *testing.T) { + expanded := expandSet(t, set) + require.NotEmpty(t, expanded.MaterializedFeatures) + + for _, member := range expanded.MaterializedFeatures { + assert.Equal(t, shorthandFeatureName, member.FeatureName) + assert.Equal(t, map[string]string{"team": "feature-store"}, member.Tags) + assert.Equal(t, "budget-1", member.BudgetPolicyId) + assert.Equal(t, notifications, member.Notifications) + assert.Equal(t, ml.MaterializedFeaturePipelineScheduleStatePaused, member.PipelineScheduleState) + } + }) + } +} + +// TestExpandMaterializationSetMembersKeepsStreamingModeOnBatchPair pins that the batch pair carries +// a trigger it has no use for rather than discarding it, so the service reports the mistake instead +// of the shorthand swallowing it. +func TestExpandMaterializationSetMembersKeepsStreamingModeOnBatchPair(t *testing.T) { + set := expandSet(t, &resources.MaterializationSet{ + BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 2) + for _, member := range set.MaterializedFeatures { + assert.Equal(t, ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm, member.StreamingMode.Mode) + } +} + +func TestExpandMaterializationSetMembersAppendsToDeclaredMembers(t *testing.T) { + declared := ml.MaterializedFeature{ + FeatureName: "main.myschema.other", + OfflineStoreConfig: shorthandOfflineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr}, + } + set := expandSet(t, &resources.MaterializationSet{ + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{declared}, + }, + SawtoothMaterializedFeatures: []ml.MaterializedFeature{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + }}, + BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ + FeatureName: "main.myschema.pair", + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr}, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 6) + assert.Equal(t, declared, set.MaterializedFeatures[0]) + assert.Nil(t, set.SawtoothMaterializedFeatures) + assert.Nil(t, set.BatchOfflineAndOnlineMaterializedFeatures) +} + +func TestExpandMaterializationSetMembersLeavesASetWithoutShorthandAlone(t *testing.T) { + declared := ml.MaterializedFeature{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr}, + } + + for name, set := range map[string]*resources.MaterializationSet{ + "no shorthand field": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{declared}, + }, + }, + "empty shorthand field": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{declared}, + }, + SawtoothMaterializedFeatures: []ml.MaterializedFeature{}, + BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{}, + }, + } { + t.Run(name, func(t *testing.T) { + expanded := expandSet(t, set) + assert.Equal(t, []ml.MaterializedFeature{declared}, expanded.MaterializedFeatures) + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2262ad6c0a5..c4c8de0f291 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -149,6 +149,12 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Expands glob patterns in pipeline library paths to include all matching files ExpandPipelineGlobPaths(), + // Reads (typed): resources.materialization_sets.*.{sawtooth,batch_offline_and_online}_materialized_features + // Updates (typed): resources.materialization_sets.*.materialized_features (appends the members each entry is made of) + // Clears (typed): both shorthand fields, after the expansion + // Expands each shorthand entry into the several materialized features one feature's materialization is made of + ExpandMaterializationSetMembers(), + // Reads (dynamic): resources.jobs.*.job_clusters (reads job clusters to merge) // Updates (dynamic): resources.jobs.*.job_clusters (merges job clusters with the same job_cluster_key) MergeJobClusters(), diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go index 147b9541386..1741c706bae 100644 --- a/bundle/config/resources/materialization_set.go +++ b/bundle/config/resources/materialization_set.go @@ -18,6 +18,19 @@ type MaterializationSet struct { BaseResource ID string `json:"id,omitempty" bundle:"readonly"` ml.MaterializationSet + + // SawtoothMaterializedFeatures declares, per entry, the whole materialization of one + // sawtooth-window feature: an offline batch member, an online batch member and an online + // streaming member. ExpandMaterializationSetMembers fans each entry out into + // MaterializedFeatures, taking the offline destination and the batch trigger for the first + // member, the online destination and the batch trigger for the second, and the online + // destination and the streaming mode for the third. + SawtoothMaterializedFeatures []ml.MaterializedFeature `json:"sawtooth_materialized_features,omitempty"` + + // BatchOfflineAndOnlineMaterializedFeatures declares, per entry, one feature materialized to + // both stores on one trigger. ExpandMaterializationSetMembers fans each entry out into two + // MaterializedFeatures differing only in their destination. + BatchOfflineAndOnlineMaterializedFeatures []ml.MaterializedFeature `json:"batch_offline_and_online_materialized_features,omitempty"` } func (s *MaterializationSet) UnmarshalJSON(b []byte) error { diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 82f3dd8fd32..04266193087 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -238,6 +238,14 @@ func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( } } +// isDerivedCronSchedule mirrors MaterializedFeatureUtils.isDerivedCronSchedule: only a cron +// schedule trigger carrying DERIVED opts out of a pinned schedule, and an unset mode reads as +// MANUAL. +func isDerivedCronSchedule(member ml.MaterializedFeature) bool { + return member.CronScheduleTrigger != nil && + member.CronScheduleTrigger.Mode == ml.CronScheduleModeDerived +} + // validateSawtoothMaterializedFeatures mirrors the rule that a sawtooth feature declares all three // of its materialized features itself. The request is not expanded for it: a materialization set // stamps a set id on every member before the backend reaches its normalization, and that @@ -281,6 +289,20 @@ func (s *FakeWorkspace) validateSawtoothMaterializedFeatures( featureName), } } + + for _, member := range byFeature[featureName] { + if member.StreamingMode != nil || isDerivedCronSchedule(member) { + continue + } + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf( + "Feature '%s' must declare a DERIVED cron schedule on its batch materialize "+ + "feature requests. The sawtooth batch materializations refresh on a daily "+ + "schedule the service computes, so a caller neither pins nor omits it.", + featureName), + } + } } return nil } From 66895760a6e8d5f31b7fca7632bd295c5bbfff3b Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 01:11:27 +0000 Subject: [PATCH 28/47] [FS-2621] Record the shorthand case in the acceptance README Co-authored-by: Isaac --- acceptance/bundle/resources/materialization_sets/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index b473ca72a72..0a8bc2f283c 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -68,6 +68,7 @@ zero with its state dropped. The deployment-history leg is the one path that rec | `batch` | a batch feature with both destinations creates and settles; neither member of its pair can be removed alone, and removing both empties the set, which the update endpoint refuses in favour of delete; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not; a set mixing a batch-path feature with a streaming-path one is refused at create | | `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing; the three members are one cascade group, so each of the three strict subsets of them is refused -- with the message depending on whether the removal names the member a delete is applied to -- and removing all three empties the set, which the update endpoint refuses | | `settled_failure` | a failure the server hits after reserving an operation comes back as HTTP 200 with the operation FAILED: the adapter recovers the error code from the operation and the per-member outcome from its metadata, the update fails naming each member left behind and writes nothing, and the destroy reports success anyway because the engine forgives a delete error it cannot classify | +| `shorthand` | a feature's whole materialization declared once in `sawtooth_materialized_features` expands into its three members, and in `batch_offline_and_online_materialized_features` into its two; neither authoring field survives into the summary, a redeploy plans nothing, and one edit to the online destination reaches both online members while the offline one stays put. A batch schedule the service refuses is reported by the service naming the feature, not by the CLI -- the expansion judges nothing it is given. The last arm mixes a shorthand entry with members written out in full, so the expansion is appended rather than substituted | | `streaming` | a streaming online member needs no schedule and no cron; declaring a schedule state on it, or repointing its destination, plans as a `recreate` rather than an in-place update; with two features materialized to one destination and no offline member between them, each online member is its own cascade group, so one can be removed alone as an `update` | ## Running against a workspace From dc11316dcaf1fbde33d4e296841fd277b7fa8321 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 03:09:58 +0000 Subject: [PATCH 29/47] [FS-2621] Count an emptied member list as an update, not a recreate Declaring materialized_features with no value empties the list as one path rather than one per member, so the membership carve-out missed it and the immutable rules classified it as a recreate. The deploy then deleted the set and the create refused the empty list, leaving the set gone. The carve-out now covers the member field itself as well as a single member. An empty list and no value reach the same endpoint and get the same refusal, which names the delete endpoint as the way to remove every member. Co-authored-by: Isaac --- .../batch/drop-both-null.yml.tmpl | 26 ++++++++++ .../materialization_sets/batch/output.txt | 32 +++++++++++- .../materialization_sets/batch/script | 10 ++++ .../direct/dresources/materialization_set.go | 15 +++--- .../dresources/materialization_set_test.go | 50 +++++++++++++++++++ 5 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl new file mode 100644 index 00000000000..8c72a5c0c3c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index 2d148021ce4..74f124bfdc4 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -9,7 +9,7 @@ Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Created features.batchfeat Created materialization_sets.set1 -Files: 9 uploaded, 0 deleted +Files: 10 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json @@ -60,6 +60,13 @@ Files: 2 uploaded, 0 deleted === Removing both satisfies the cascade but empties the set, which the update endpoint refuses === in favour of the delete endpoint. Restoring the pair plans nothing, so no refusal wrote. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) @@ -69,7 +76,28 @@ HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. -Files: 1 uploaded, 0 deleted +Files: 2 uploaded, 0 deleted + +=== Writing no value instead of an empty list is the same declaration, and has to reach the same +=== endpoint: the member field emptied whole is one path rather than one per member, and the +=== immutable rules match it just as readily. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script index cdbb6b2ea7f..bc007af0f2c 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/script +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -46,6 +46,16 @@ trace musterr $CLI bundle deploy --auto-approve title "Removing both satisfies the cascade but empties the set, which the update endpoint refuses" title "in favour of the delete endpoint. Restoring the pair plans nothing, so no refusal wrote." render drop-both.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "Writing no value instead of an empty list is the same declaration, and has to reach the same" +title "endpoint: the member field emptied whole is one path rather than one per member, and the" +title "immutable rules match it just as readily." +render drop-both-null.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions trace musterr $CLI bundle deploy --auto-approve render databricks.yml.tmpl trace $CLI bundle plan -o json > out.plan.json diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 6733ad67d32..f2f5f43f0f0 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -118,13 +118,14 @@ func isAbsentRemotely(remote any) bool { // destroy-and-recreate, and it suppresses the drift left behind by an ignore_remote_changes rule // that cannot fire. func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { - // A member added to or removed from the set is a two-segment path, - // materialized_features[member='||'], which an - // immutable field rule under materialized_features[*] still matches. That match counts as an - // update rather than a destroy-and-recreate, because removing a member is what the update - // endpoint reconciles. An addition reaches the same endpoint and is refused there, naming the - // feature that matched no member. - if path.Len() == 2 && path.HasPatternPrefix(materializedFeaturesPattern) { + // A member added or removed whole is a two-segment path, + // materialized_features[member='||'], and the + // member list emptied whole is the one-segment materialized_features. An immutable field rule + // under materialized_features[*] matches either, because the matcher truncates a pattern longer + // than the path. Both count as an update rather than a destroy-and-recreate, because + // reconciling membership is what the update endpoint does: it refuses an addition by naming the + // feature that matched no member, and an emptied list by pointing at the delete endpoint. + if path.Len() <= 2 && path.HasPatternPrefix(materializedFeaturesPattern) { change.Action = deployplan.Update return nil } diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go index cfe46ff17d7..5ef3a0b9d9b 100644 --- a/bundle/direct/dresources/materialization_set_test.go +++ b/bundle/direct/dresources/materialization_set_test.go @@ -4,8 +4,11 @@ import ( "errors" "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMaterializationSetMemberKey(t *testing.T) { @@ -145,3 +148,50 @@ func TestMaterializationSetChangeFailuresWithoutFailedMembers(t *testing.T) { })) assert.Empty(t, materializationSetChangeFailures(nil)) } + +// TestMaterializationSetMembershipChangeIsAnUpdate enumerates every path shape the planner hands +// this resource, so the membership carve-out's reach is visible rather than inferred. The two +// shapes it covers are a member added or removed whole and the member list emptied whole; every +// other path keeps the action the rules gave it. +func TestMaterializationSetMembershipChangeIsAnUpdate(t *testing.T) { + const member = "materialized_features[member='main.ns.feat|store|false']" + + for _, test := range []struct { + path string + action deployplan.ActionType + }{ + {"materialized_features", deployplan.Update}, + {member, deployplan.Update}, + {"materialized_features[member='main.ns.feat||false']", deployplan.Update}, + {member + ".feature_name", deployplan.Recreate}, + {member + ".cron_schedule_trigger", deployplan.Recreate}, + {member + ".online_store_config.table_name_prefix", deployplan.Recreate}, + {"materialization_set_id", deployplan.Recreate}, + } { + t.Run(test.path, func(t *testing.T) { + path, err := structpath.ParsePath(test.path) + require.NoError(t, err) + + change := &ChangeDesc{Action: deployplan.Recreate, Reason: "immutable"} + require.NoError(t, (&ResourceMaterializationSet{}).OverrideChangeDesc( + t.Context(), path, change, nil)) + assert.Equal(t, test.action, change.Action) + }) + } +} + +// TestMaterializationSetDestinationSuppressionStillReachesMemberFields keeps the two blocks of +// OverrideChangeDesc disjoint: widening the membership carve-out to one segment must not shadow the +// input-only suppression, which only ever sees a member's destination three segments down. +func TestMaterializationSetDestinationSuppressionStillReachesMemberFields(t *testing.T) { + path, err := structpath.ParsePath( + "materialized_features[member='main.ns.feat||false'].offline_store_config") + require.NoError(t, err) + + change := &ChangeDesc{Action: deployplan.Recreate, Reason: "spec:immutable", New: "declared"} + require.NoError(t, (&ResourceMaterializationSet{}).OverrideChangeDesc( + t.Context(), path, change, nil)) + + assert.Equal(t, deployplan.Skip, change.Action) + assert.Equal(t, "input_only_not_returned", change.Reason) +} From 32bde0580eace3845701971726e07dac4de2cb9a Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 03:19:27 +0000 Subject: [PATCH 30/47] [FS-2621] Pin the route the empty-set refusal names The update endpoint answers an emptied member list with "Delete the set instead", and nothing exercised that answer: every delete in the goldens came from bundle destroy, which is the destroy plan rather than the deploy path's delete. Dropping the set resource now has an arm asserting it plans as a delete and settles. Co-authored-by: Isaac --- .../batch/no-set.yml.tmpl | 22 ++++++++++++ .../materialization_sets/batch/output.txt | 35 ++++++++++++++++++- .../materialization_sets/batch/script | 11 ++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl diff --git a/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl new file mode 100644 index 00000000000..ed86dd96e53 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index 74f124bfdc4..f80f9605368 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -9,7 +9,7 @@ Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Created features.batchfeat Created materialization_sets.set1 -Files: 10 uploaded, 0 deleted +Files: 11 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json @@ -106,6 +106,39 @@ Files: 2 uploaded, 0 deleted "resources.materialization_sets.set1": "skip" } +=== Taking the refusal's advice: dropping the set resource rather than its members plans as a +=== delete and settles. This is the deploy path's delete, not bundle destroy's. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "delete" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + delete resources.materialization_sets.set1 +Deleted materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "create" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + === Repointing the destinations is a recreate, not a silent no-op. The server cannot move a === materialization in place, so the set is rebuilt at the new location. >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script index bc007af0f2c..62624214c43 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/script +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -61,6 +61,17 @@ render databricks.yml.tmpl trace $CLI bundle plan -o json > out.plan.json trace set_actions +title "Taking the refusal's advice: dropping the set resource rather than its members plans as a" +title "delete and settles. This is the deploy path's delete, not bundle destroy's." +render no-set.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy + title "Repointing the destinations is a recreate, not a silent no-op. The server cannot move a" title "materialization in place, so the set is rebuilt at the new location." render moved.yml.tmpl From 859898a39323438545009c745e9ec777d6e82007 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 04:50:41 +0000 Subject: [PATCH 31/47] [FS-2621] Record every cloud case passing against a workspace Two claims had never been settled live: the shorthand case had only run against the fake, and batch carried two arms added after the last cloud run - the null member-list spelling and the drop-the-resource delete route. Both pass against the swap with no golden changed, so the emptied-set refusal is the service's own text. Two README facts were stale and one cost a run: the cloud-case inventory omitted shorthand, and the MATSET_STREAM default names a Stream that has since been deleted, which fails the run at the feature create. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 0a8bc2f283c..6a2c0ae39fd 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -73,8 +73,11 @@ zero with its state dropped. The deployment-history leg is the one path that rec ## Running against a workspace -`batch`, `sawtooth` and `streaming` carry `Cloud = true`. All three reference an existing `Stream`, -because a Stream is not a bundle resource; override `MATSET_CATALOG`, `MATSET_SCHEMA`, +`batch`, `sawtooth`, `streaming` and `shorthand` reference an existing `Stream`, because a Stream +is not a bundle resource. **The `MATSET_STREAM` default names one developer's Stream and goes stale +when it is deleted**, which fails the run at the feature create with `Stream '...' not found`; list +what the workspace has with `GET /api/2.0/feature-engineering/streams?parent=.` and +override. Override `MATSET_CATALOG`, `MATSET_SCHEMA`, `MATSET_ONLINE_STORE` and `MATSET_STREAM` to move them to another workspace, and set `MATSET_PROFILE` to pin the bundle to a named profile. **`DATABRICKS_CONFIG_PROFILE` does not reach the CLI subprocess**, and the harness builds its own workspace client from the ambient environment @@ -89,8 +92,8 @@ does run, because the fake only requires the key to carry a prefix at all. ### Running a cloud leg -Six cases carry `Cloud = true`: `batch`, `sawtooth`, `streaming`, `immutable`, `membership` and -`removal_scope`. `basic`, `bind`, `destination` and `schedule_state` stay local, because every +Seven cases carry `Cloud = true`: `batch`, `sawtooth`, `streaming`, `immutable`, `membership`, +`removal_scope` and `shorthand`. `basic`, `bind`, `destination` and `schedule_state` stay local, because every server-dependent claim they make is already exercised by `batch`. The cases default to the shared acceptance schema; override `MATSET_CATALOG`, `MATSET_SCHEMA`, @@ -214,6 +217,28 @@ now count what got through and point at the operation's progress), and two refus HTTP 400 to a settled FAILED operation. The fake and four goldens were corrected from what the live run returned. +**Fifth build, universe `92bd718e722a0`** -- the tip of `stack/fs2680-update-set-lro`, run through +the `daniel-bai-fs2680` swap on 2026-10-23. All seven `Cloud = true` cases pass with no golden +changed, run one at a time, as does `features/lifecycle` on its `DMS=` leg. Two of them had claims +nothing live had settled: `shorthand` had never run against a workspace at all, and `batch` carried +two arms -- the null member-list spelling and the drop-the-resource delete route -- that had only +run against the fake. So the emptied-set refusal's "Delete the set instead" is the service's own +text, and the one-segment `materialized_features` path classifies as an update on the live API and +not just in the unit tests. + +`shorthand` is the case that run first justified: a sawtooth entry creates the three members the +service wants, a redeploy plans nothing, one edit to the online destination recreates exactly the +two online members, a shorthand entry and members written out in full coexist in one set, and the +DERIVED-schedule refusal its first arm asserts is the service's own text. + +The swap's own log is what proves the routing: during the 63-second run it served 5 +`CreateMaterializationSetRequest`, 7 `DeleteMaterializationSetRequest`, 6 +`GetMaterializationSetRequest`, 15 `ListMaterializationSetMembersRequest`, 3 +`CreateMaterializationSetMembersRequest` and 4 `DeleteMaterializedFeatureRequest`, interleaved with +this run's own fixture names. Zero `UpdateMaterializationSetRequest`, which is the case's one +coverage limit: every membership edit it makes is to an immutable field, so it plans a recreate and +never reaches the update endpoint. `batch`'s emptied-member arms are what exercise that. + ## Verified against a workspace, with no case of their own Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once From 17dd64cc3de7ed70c9a8950d9e0f949f2d33a219 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 08:15:57 +0000 Subject: [PATCH 32/47] [FS-2621] Report that a materialization set has no workspace URL bundle summary rendered "URL: (not deployed)" for a set that was in fact deployed. BaseResource.GetURL returns ("", true), which the summary reads as a supported type awaiting deployment; a set has no standalone page, so it belongs in the ("", false) case that drops the line entirely. Co-authored-by: Isaac --- .../config/resources/materialization_set.go | 7 +++++- .../resources/materialization_set_test.go | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 bundle/config/resources/materialization_set_test.go diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go index 1741c706bae..8c32b1058a6 100644 --- a/bundle/config/resources/materialization_set.go +++ b/bundle/config/resources/materialization_set.go @@ -73,6 +73,11 @@ func (s *MaterializationSet) GetName() string { return "" } -// InitializeURL leaves URL unset because a set has no standalone page in the workspace; its +// GetURL reports that this type has no workspace URL: a set has no standalone page, and its // members surface under their own features. +func (s *MaterializationSet) GetURL() (string, bool) { + return "", false +} + +// InitializeURL leaves URL unset. See GetURL. func (s *MaterializationSet) InitializeURL(_ url.URL) {} diff --git a/bundle/config/resources/materialization_set_test.go b/bundle/config/resources/materialization_set_test.go new file mode 100644 index 00000000000..643ceac4a7c --- /dev/null +++ b/bundle/config/resources/materialization_set_test.go @@ -0,0 +1,25 @@ +package resources + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestMaterializationSetReportsNoWorkspaceURL pins the signal that keeps `bundle summary` from +// printing a URL line for a set. BaseResource.GetURL returns ("", true), which the summary reads as +// a supported type awaiting deployment and renders as "(not deployed)". +func TestMaterializationSetReportsNoWorkspaceURL(t *testing.T) { + set := &MaterializationSet{ID: "0123456789abcdef0123456789abcdef"} + set.URL = "https://example.cloud.databricks.com/explore/data/features" + + rawURL, supportsURL := set.GetURL() + assert.False(t, supportsURL) + assert.Empty(t, rawURL) + + set.InitializeURL(url.URL{Scheme: "https", Host: "example.cloud.databricks.com"}) + rawURL, supportsURL = set.GetURL() + assert.False(t, supportsURL, "InitializeURL must not make the type look URL-bearing") + assert.Empty(t, rawURL) +} From b40016fc3f9eaa92549d67201fa8840b53c98af4 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 18:23:02 +0000 Subject: [PATCH 33/47] [FS-2621] Label the member key and omit an absent online store The key flattened an absent online destination and an online destination with no store name to the same empty component, so two distinct members shared a key. The component is now omitted when the online config is absent, which separates them; the server already distinguishes the two by holding the name in an Option. Each component is labelled, so a member path in plan output names what it is keyed on rather than reading as a bare positional join. Co-authored-by: Isaac --- .../materialization_sets/batch/output.txt | 2 +- .../materialization_sets/bind/output.txt | 8 ++-- .../destination/output.txt | 38 +++++++++---------- .../materialization_sets/shorthand/output.txt | 4 +- .../direct/dresources/materialization_set.go | 33 ++++++++-------- .../dresources/materialization_set_test.go | 31 ++++++++++++--- 6 files changed, 68 insertions(+), 48 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index f80f9605368..a2fe2d0890e 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -149,8 +149,8 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged } >>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)" out.plan.json -spec:immutable old=batch_[UNIQUE_NAME] new=moved_[UNIQUE_NAME] spec:immutable old=batch_[UNIQUE_NAME]_off new=moved_[UNIQUE_NAME]_off +spec:immutable old=batch_[UNIQUE_NAME] new=moved_[UNIQUE_NAME] === The schedule state is write-once, so pausing the feature is a recreate rather than the === in-place update the endpoint would refuse diff --git a/acceptance/bundle/resources/materialization_sets/bind/output.txt b/acceptance/bundle/resources/materialization_sets/bind/output.txt index 807be1954f4..a5133235426 100644 --- a/acceptance/bundle/resources/materialization_sets/bind/output.txt +++ b/acceptance/bundle/resources/materialization_sets/bind/output.txt @@ -33,10 +33,10 @@ Run 'bundle deploy' to deploy changes to your workspace } >>> destination_changes -materialized_features[member='main.myschema.bindfeat|danielb|false'].online_store_config.catalog_name skip input_only_not_returned -materialized_features[member='main.myschema.bindfeat|danielb|false'].online_store_config.schema_name skip input_only_not_returned -materialized_features[member='main.myschema.bindfeat|danielb|false'].online_store_config.table_name_prefix skip input_only_not_returned -materialized_features[member='main.myschema.bindfeat||false'].offline_store_config skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|is_streaming_mode=false'].offline_store_config skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix skip input_only_not_returned === And the deploy is a no-op rather than a rebuild >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/materialization_sets/destination/output.txt b/acceptance/bundle/resources/materialization_sets/destination/output.txt index 6b5aca32954..dae6c56dd9a 100644 --- a/acceptance/bundle/resources/materialization_sets/destination/output.txt +++ b/acceptance/bundle/resources/materialization_sets/destination/output.txt @@ -14,19 +14,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "action": "skip", "store_config_changes": { - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.catalog_name": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.schema_name": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.table_name_prefix": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat||false'].offline_store_config": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { "action": "skip", "reason": "input_only_not_returned" } @@ -47,19 +47,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "action": "recreate", "store_config_changes": { - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.catalog_name": { - "action": "recreate", - "reason": "spec:immutable" - }, - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.schema_name": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.table_name_prefix": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat||false'].offline_store_config": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { "action": "skip", "reason": "input_only_not_returned" } @@ -77,25 +77,25 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "action": "recreate", "store_config_changes": { - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.catalog_name": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.schema_name": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config.table_name_prefix": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat|danielb|false'].online_store_config.table_name_prefix": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { "action": "skip", "reason": "input_only_not_returned" }, - "materialized_features[member='main.myschema.destfeat||false'].offline_store_config": { + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { "action": "skip", "reason": "input_only_not_returned" - }, - "materialized_features[member='main.myschema.destfeat||false'].offline_store_config.table_name_prefix": { - "action": "recreate", - "reason": "spec:immutable" } } } diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/output.txt b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt index f449c3a184c..e1fc72dff49 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/output.txt +++ b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt @@ -58,8 +58,8 @@ skip recreate >>> recreated_fields -materialized_features[member='danielb.fs2680.shd_[UNIQUE_NAME]|danielb|false'].online_store_config.table_name_prefix -materialized_features[member='danielb.fs2680.shd_[UNIQUE_NAME]|danielb|true'].online_store_config.table_name_prefix +materialized_features[member='feature_name=danielb.fs2680.shd_[UNIQUE_NAME]|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix +materialized_features[member='feature_name=danielb.fs2680.shd_[UNIQUE_NAME]|online_store_name=danielb|is_streaming_mode=true'].online_store_config.table_name_prefix >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index f2f5f43f0f0..7e43a8230e9 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -44,24 +44,25 @@ func escapeMemberKeyPart(part string) string { } // materializationSetMemberKey identifies a materialized feature member by content rather than -// position. The key components are the feature name, the online store name (empty for the offline -// member), and whether the trigger is streaming. +// position, as labelled components joined by the separator escapeMemberKeyPart escapes. // -// The server builds a key from the same three components, in the same order, in -// MaterializationSetMemberKey.keyOf -- see +// online_store_name is omitted rather than left empty when the member has no online destination, +// which is what keeps an offline member distinct from an online one whose store name is unset. +// +// The server pairs members by the same components in MaterializationSetMemberKey -- see // feature-store/src/main/scala/com/databricks/feature-store/utils/materialized-features/MaterializationSetUtils.scala -// -- and pairs a requested member with a stored one by it. Adding, dropping or redefining a -// component here takes the same change there. +// -- holding them in a case class rather than a joined string, so this encoding is the CLI's own +// and only the component list has to agree. Adding, dropping or redefining one takes the same +// change there. func materializationSetMemberKey(member ml.MaterializedFeature) (string, string) { - onlineStoreName := "" + parts := []string{"feature_name=" + escapeMemberKeyPart(member.FeatureName)} if member.OnlineStoreConfig != nil { - onlineStoreName = member.OnlineStoreConfig.OnlineStoreName + parts = append(parts, + "online_store_name="+escapeMemberKeyPart(member.OnlineStoreConfig.OnlineStoreName)) } - return "member", strings.Join([]string{ - escapeMemberKeyPart(member.FeatureName), - escapeMemberKeyPart(onlineStoreName), - strconv.FormatBool(member.StreamingMode != nil), - }, "|") + parts = append(parts, "is_streaming_mode="+strconv.FormatBool(member.StreamingMode != nil)) + + return "member", strings.Join(parts, "|") } func (*ResourceMaterializationSet) KeyedSlices() map[string]any { @@ -118,9 +119,9 @@ func isAbsentRemotely(remote any) bool { // destroy-and-recreate, and it suppresses the drift left behind by an ignore_remote_changes rule // that cannot fire. func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { - // A member added or removed whole is a two-segment path, - // materialized_features[member='||'], and the - // member list emptied whole is the one-segment materialized_features. An immutable field rule + // A member added or removed whole is a two-segment path, materialized_features[member=...] + // carrying the key materializationSetMemberKey builds, and the member list emptied whole is the + // one-segment materialized_features. An immutable field rule // under materialized_features[*] matches either, because the matcher truncates a pattern longer // than the path. Both count as an update rather than a destroy-and-recreate, because // reconciling membership is what the update endpoint does: it refuses an addition by naming the diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go index 5ef3a0b9d9b..318e9e966d7 100644 --- a/bundle/direct/dresources/materialization_set_test.go +++ b/bundle/direct/dresources/materialization_set_test.go @@ -28,13 +28,13 @@ func TestMaterializationSetMemberKey(t *testing.T) { field, value := materializationSetMemberKey(offline) assert.Equal(t, "member", field) - assert.Equal(t, "main.default.feat||false", value) + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=false", value) _, onlineValue := materializationSetMemberKey(online) - assert.Equal(t, "main.default.feat|store_one|false", onlineValue) + assert.Equal(t, "feature_name=main.default.feat|online_store_name=store_one|is_streaming_mode=false", onlineValue) _, streamingValue := materializationSetMemberKey(streaming) - assert.Equal(t, "main.default.feat||true", streamingValue) + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=true", streamingValue) // The three members of one feature must not collide with each other. assert.NotEqual(t, value, onlineValue) @@ -66,6 +66,25 @@ func TestMaterializationSetMemberKeyEscapesSeparator(t *testing.T) { assert.NotEqual(t, first, forged) } +// An absent online destination and an online destination carrying no store name are distinct +// members, which is why the component is omitted rather than left empty. +func TestMaterializationSetMemberKeySeparatesAbsentOnlineStoreFromEmptyName(t *testing.T) { + _, offline := materializationSetMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + }) + _, onlineWithoutName := materializationSetMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{CatalogName: "main", SchemaName: "default"}, + }) + + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=false", offline) + assert.Equal(t, + "feature_name=main.default.feat|online_store_name=|is_streaming_mode=false", + onlineWithoutName) + assert.NotEqual(t, offline, onlineWithoutName) +} + // operationStub stands in for a generated operation interface. The three verbs' interfaces differ // only in what Wait returns, which materializationSetOperationTarget never calls. type operationStub struct { @@ -154,7 +173,7 @@ func TestMaterializationSetChangeFailuresWithoutFailedMembers(t *testing.T) { // shapes it covers are a member added or removed whole and the member list emptied whole; every // other path keeps the action the rules gave it. func TestMaterializationSetMembershipChangeIsAnUpdate(t *testing.T) { - const member = "materialized_features[member='main.ns.feat|store|false']" + const member = "materialized_features[member='feature_name=main.ns.feat|online_store_name=store|is_streaming_mode=false']" for _, test := range []struct { path string @@ -162,7 +181,7 @@ func TestMaterializationSetMembershipChangeIsAnUpdate(t *testing.T) { }{ {"materialized_features", deployplan.Update}, {member, deployplan.Update}, - {"materialized_features[member='main.ns.feat||false']", deployplan.Update}, + {"materialized_features[member='feature_name=main.ns.feat|is_streaming_mode=false']", deployplan.Update}, {member + ".feature_name", deployplan.Recreate}, {member + ".cron_schedule_trigger", deployplan.Recreate}, {member + ".online_store_config.table_name_prefix", deployplan.Recreate}, @@ -185,7 +204,7 @@ func TestMaterializationSetMembershipChangeIsAnUpdate(t *testing.T) { // input-only suppression, which only ever sees a member's destination three segments down. func TestMaterializationSetDestinationSuppressionStillReachesMemberFields(t *testing.T) { path, err := structpath.ParsePath( - "materialized_features[member='main.ns.feat||false'].offline_store_config") + "materialized_features[member='feature_name=main.ns.feat|is_streaming_mode=false'].offline_store_config") require.NoError(t, err) change := &ChangeDesc{Action: deployplan.Recreate, Reason: "spec:immutable", New: "declared"} From ee4cf0b5d541139d3847cec3885da2bc63ac9d9e Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 19:21:10 +0000 Subject: [PATCH 34/47] [FS-2621] Name the mutator for what it expands, and trim its comments Review feedback: members -> MaterializedFeatureMembers throughout, since a set's members are materialized features; and the doc comments stated more than the reader needs, several of them restating server behaviour that belongs in the pull request rather than beside the code. Co-authored-by: Isaac --- ...ation_set_materialized_feature_members.go} | 51 ++++++++++--------- ...rialized_feature_members_coverage_test.go} | 6 +-- ..._set_materialized_feature_members_test.go} | 16 +++--- .../resourcemutator/resource_mutator.go | 2 +- .../config/resources/materialization_set.go | 13 ++--- .../configs/materialization_sets.yml | 8 --- .../direct/dresources/materialization_set.go | 34 ++++++------- 7 files changed, 61 insertions(+), 69 deletions(-) rename bundle/config/mutator/resourcemutator/{expand_materialization_set_members.go => expand_materialization_set_materialized_feature_members.go} (59%) rename bundle/config/mutator/resourcemutator/{expand_materialization_set_members_coverage_test.go => expand_materialization_set_materialized_feature_members_coverage_test.go} (93%) rename bundle/config/mutator/resourcemutator/{expand_materialization_set_members_test.go => expand_materialization_set_materialized_feature_members_test.go} (91%) diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_members.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go similarity index 59% rename from bundle/config/mutator/resourcemutator/expand_materialization_set_members.go rename to bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go index 013b0745c29..d6201542eb6 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_members.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go @@ -8,25 +8,26 @@ import ( "github.com/databricks/databricks-sdk-go/service/ml" ) -type expandMaterializationSetMembers struct{} +type expandMaterializationSetMaterializedFeatureMembers struct{} -func ExpandMaterializationSetMembers() bundle.Mutator { - return &expandMaterializationSetMembers{} +func ExpandMaterializationSetMaterializedFeatureMembers() bundle.Mutator { + return &expandMaterializationSetMaterializedFeatureMembers{} } -func (m *expandMaterializationSetMembers) Name() string { - return "ExpandMaterializationSetMembers" +func (m *expandMaterializationSetMaterializedFeatureMembers) Name() string { + return "ExpandMaterializationSetMaterializedFeatureMembers" } -// sawtoothMembers builds the three materialized features a sawtooth-window feature is made of. The -// destination and the trigger are split across them; every other field the entry carries is held by -// all three. +// sawtoothMaterializedFeatureMembers builds the three materialized features a sawtooth-window +// feature needs to declare for its materialization: a batch offline materialized feature, a batch +// online materialized feature, and a streaming online materialized feature, where the batch offline +// and batch online use a DERIVED cron schedule and the streaming online and batch online must have +// the same online destination. // -// materialized_feature_id is deliberately absent: the server assigns one per member, so a single -// declared id cannot belong to three. ForceSendFields is absent because the dyn-to-typed -// conversion rebuilds it per member from the fields that member holds. -func sawtoothMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { +// materialized_feature_id is absent here because it is assigned server side. +func sawtoothMaterializedFeatureMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { return []ml.MaterializedFeature{ + // Batch offline. { FeatureName: entry.FeatureName, Tags: entry.Tags, @@ -38,6 +39,7 @@ func sawtoothMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { CronScheduleTrigger: entry.CronScheduleTrigger, TableTrigger: entry.TableTrigger, }, + // Batch online. { FeatureName: entry.FeatureName, Tags: entry.Tags, @@ -49,6 +51,7 @@ func sawtoothMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { CronScheduleTrigger: entry.CronScheduleTrigger, TableTrigger: entry.TableTrigger, }, + // Streaming online. { FeatureName: entry.FeatureName, Tags: entry.Tags, @@ -61,13 +64,11 @@ func sawtoothMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { } } -// batchOfflineAndOnlineMembers builds the two materialized features a feature materialized to both -// stores is made of. Only the destination is split; every other field the entry carries, trigger -// included, is held by both. -// -// The omissions are the same as in sawtoothMembers. -func batchOfflineAndOnlineMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { +// batchOfflineAndOnlineMaterializedFeatureMembers builds two materializations of the specified +// feature to the online and offline destination. +func batchOfflineAndOnlineMaterializedFeatureMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { return []ml.MaterializedFeature{ + // Batch offline. { FeatureName: entry.FeatureName, Tags: entry.Tags, @@ -80,6 +81,7 @@ func batchOfflineAndOnlineMembers(entry ml.MaterializedFeature) []ml.Materialize TableTrigger: entry.TableTrigger, StreamingMode: entry.StreamingMode, }, + // Batch online. { FeatureName: entry.FeatureName, Tags: entry.Tags, @@ -95,19 +97,22 @@ func batchOfflineAndOnlineMembers(entry ml.MaterializedFeature) []ml.Materialize } } -func (m *expandMaterializationSetMembers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *expandMaterializationSetMaterializedFeatureMembers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for _, set := range b.Config.Resources.MaterializationSets { if set == nil { continue } - // Appended rather than assigned: a set may declare one feature in shorthand and another - // as members, and the adapter keys members by content rather than by position. + // Append rather than assign because a set can declare one feature in shorthand and another + // as individual materialized feature members. The adapter keys materialized feature members + // by content rather than by position. for _, entry := range set.SawtoothMaterializedFeatures { - set.MaterializedFeatures = append(set.MaterializedFeatures, sawtoothMembers(entry)...) + set.MaterializedFeatures = append(set.MaterializedFeatures, + sawtoothMaterializedFeatureMembers(entry)...) } for _, entry := range set.BatchOfflineAndOnlineMaterializedFeatures { - set.MaterializedFeatures = append(set.MaterializedFeatures, batchOfflineAndOnlineMembers(entry)...) + set.MaterializedFeatures = append(set.MaterializedFeatures, + batchOfflineAndOnlineMaterializedFeatureMembers(entry)...) } set.SawtoothMaterializedFeatures = nil diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go similarity index 93% rename from bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go rename to bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go index 9ec54471ffa..bd4b5ce6275 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_members_coverage_test.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go @@ -68,10 +68,10 @@ func memberWithEveryFieldSet(t *testing.T) ml.MaterializedFeature { return member } -// TestExpandMaterializationSetMembersCoversEveryMemberField keeps each expansion's field list +// TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField keeps each expansion's field list // exhaustive. Members are built by naming their fields rather than by copying the entry, so a // member field added to the proto later reaches no expanded member and nothing else notices. -func TestExpandMaterializationSetMembersCoversEveryMemberField(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField(t *testing.T) { entry := memberWithEveryFieldSet(t) for name, set := range map[string]*resources.MaterializationSet{ @@ -107,7 +107,7 @@ func TestExpandMaterializationSetMembersCoversEveryMemberField(t *testing.T) { } assert.Truef(t, carried, "field %s reached no member of the %s expansion. Add it to the member "+ - "literals in expand_materialization_set_members.go, or to "+ + "literals in expand_materialization_set_materialized_feature_members.go, or to "+ "memberFieldsNotCarried with the reason it is server-owned", field.Name, name) } diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go similarity index 91% rename from bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go rename to bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go index df46c667616..f9e61d170c8 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_members_test.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go @@ -44,13 +44,13 @@ func expandSet(t *testing.T, set *resources.MaterializationSet) *resources.Mater }, }, } - diags := bundle.Apply(t.Context(), b, resourcemutator.ExpandMaterializationSetMembers()) + diags := bundle.Apply(t.Context(), b, resourcemutator.ExpandMaterializationSetMaterializedFeatureMembers()) require.NoError(t, diags.Error()) return b.Config.Resources.MaterializationSets["set1"] } -func TestExpandMaterializationSetMembersSawtooth(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersSawtooth(t *testing.T) { set := expandSet(t, &resources.MaterializationSet{ MaterializationSet: ml.MaterializationSet{}, SawtoothMaterializedFeatures: []ml.MaterializedFeature{{ @@ -88,7 +88,7 @@ func TestExpandMaterializationSetMembersSawtooth(t *testing.T) { assert.Nil(t, set.SawtoothMaterializedFeatures) } -func TestExpandMaterializationSetMembersBatchOfflineAndOnline(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersBatchOfflineAndOnline(t *testing.T) { trigger := &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr} set := expandSet(t, &resources.MaterializationSet{ BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ @@ -115,7 +115,7 @@ func TestExpandMaterializationSetMembersBatchOfflineAndOnline(t *testing.T) { assert.Nil(t, set.BatchOfflineAndOnlineMaterializedFeatures) } -func TestExpandMaterializationSetMembersCarriesSharedFields(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersCarriesSharedFields(t *testing.T) { notifications := &ml.MaterializationNotifications{ OnFailure: &ml.MaterializationFailureNotification{EmailAddresses: []string{"a@example.com"}}, } @@ -150,10 +150,10 @@ func TestExpandMaterializationSetMembersCarriesSharedFields(t *testing.T) { } } -// TestExpandMaterializationSetMembersKeepsStreamingModeOnBatchPair pins that the batch pair carries +// TestExpandMaterializationSetMaterializedFeatureMembersKeepsStreamingModeOnBatchPair pins that the batch pair carries // a trigger it has no use for rather than discarding it, so the service reports the mistake instead // of the shorthand swallowing it. -func TestExpandMaterializationSetMembersKeepsStreamingModeOnBatchPair(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersKeepsStreamingModeOnBatchPair(t *testing.T) { set := expandSet(t, &resources.MaterializationSet{ BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ FeatureName: shorthandFeatureName, @@ -169,7 +169,7 @@ func TestExpandMaterializationSetMembersKeepsStreamingModeOnBatchPair(t *testing } } -func TestExpandMaterializationSetMembersAppendsToDeclaredMembers(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersAppendsToDeclaredMembers(t *testing.T) { declared := ml.MaterializedFeature{ FeatureName: "main.myschema.other", OfflineStoreConfig: shorthandOfflineStore(), @@ -200,7 +200,7 @@ func TestExpandMaterializationSetMembersAppendsToDeclaredMembers(t *testing.T) { assert.Nil(t, set.BatchOfflineAndOnlineMaterializedFeatures) } -func TestExpandMaterializationSetMembersLeavesASetWithoutShorthandAlone(t *testing.T) { +func TestExpandMaterializationSetMaterializedFeatureMembersLeavesASetWithoutShorthandAlone(t *testing.T) { declared := ml.MaterializedFeature{ FeatureName: shorthandFeatureName, OfflineStoreConfig: shorthandOfflineStore(), diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index c4c8de0f291..c14aa127576 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -153,7 +153,7 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Updates (typed): resources.materialization_sets.*.materialized_features (appends the members each entry is made of) // Clears (typed): both shorthand fields, after the expansion // Expands each shorthand entry into the several materialized features one feature's materialization is made of - ExpandMaterializationSetMembers(), + ExpandMaterializationSetMaterializedFeatureMembers(), // Reads (dynamic): resources.jobs.*.job_clusters (reads job clusters to merge) // Updates (dynamic): resources.jobs.*.job_clusters (merges job clusters with the same job_cluster_key) diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go index 8c32b1058a6..a9dec0bcd53 100644 --- a/bundle/config/resources/materialization_set.go +++ b/bundle/config/resources/materialization_set.go @@ -12,8 +12,7 @@ import ( ) // MaterializationSet is the bundle config for a group of materialized features created and -// managed as one unit. Members are addressed by content rather than by position, so the set -// is reconciled against its desired membership rather than updated field by field. +// managed as one unit. type MaterializationSet struct { BaseResource ID string `json:"id,omitempty" bundle:"readonly"` @@ -21,15 +20,13 @@ type MaterializationSet struct { // SawtoothMaterializedFeatures declares, per entry, the whole materialization of one // sawtooth-window feature: an offline batch member, an online batch member and an online - // streaming member. ExpandMaterializationSetMembers fans each entry out into - // MaterializedFeatures, taking the offline destination and the batch trigger for the first - // member, the online destination and the batch trigger for the second, and the online - // destination and the streaming mode for the third. + // streaming member. ExpandMaterializationSetMaterializedFeatureMembers fans each entry out + // into three materialized features. SawtoothMaterializedFeatures []ml.MaterializedFeature `json:"sawtooth_materialized_features,omitempty"` // BatchOfflineAndOnlineMaterializedFeatures declares, per entry, one feature materialized to - // both stores on one trigger. ExpandMaterializationSetMembers fans each entry out into two - // MaterializedFeatures differing only in their destination. + // the online and offline destination. ExpandMaterializationSetMaterializedFeatureMembers fans + // each entry out into two MaterializedFeatures. BatchOfflineAndOnlineMaterializedFeatures []ml.MaterializedFeature `json:"batch_offline_and_online_materialized_features,omitempty"` } diff --git a/bundle/direct/dresources/configs/materialization_sets.yml b/bundle/direct/dresources/configs/materialization_sets.yml index 9034678d9b7..992130fec37 100644 --- a/bundle/direct/dresources/configs/materialization_sets.yml +++ b/bundle/direct/dresources/configs/materialization_sets.yml @@ -1,11 +1,3 @@ -# The set update reconciles membership and nothing else, so a member's own fields are write-once. -# The spec leaves them OPTIONAL rather than IMMUTABLE because UpdateMaterializedFeature can still -# change some of them on a member addressed directly, so materialization_sets.generated.yml will -# never cover them and these entries are permanent. -# -# A member added or removed whole also matches these patterns, because the matcher truncates a -# pattern longer than the path. ResourceMaterializationSet.OverrideChangeDesc downgrades that case -# to update so reconciling membership does not destroy the siblings the config left alone. recreate_on_changes: - field: materialized_features[*].feature_name reason: immutable diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index 7e43a8230e9..04dae74d162 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -30,8 +30,7 @@ func (*ResourceMaterializationSet) PrepareState(input *resources.Materialization return &input.MaterializationSet } -// materializationSetUpdatableFields is the only mask the update endpoint accepts, so "*" cannot -// be used. Membership is reconciled whole: the server deletes members the request omits. +// materializationSetUpdatableFields is the only mask the update endpoint accepts. var materializationSetUpdatableFields = []string{"materialized_features"} // escapeMemberKeyPart percent-escapes the separator so the join in materializationSetMemberKey is @@ -124,16 +123,15 @@ func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *s // one-segment materialized_features. An immutable field rule // under materialized_features[*] matches either, because the matcher truncates a pattern longer // than the path. Both count as an update rather than a destroy-and-recreate, because - // reconciling membership is what the update endpoint does: it refuses an addition by naming the - // feature that matched no member, and an emptied list by pointing at the delete endpoint. + // reconciling membership is what the update endpoint does. if path.Len() <= 2 && path.HasPatternPrefix(materializedFeaturesPattern) { change.Action = deployplan.Update return nil } // The ignore_remote_changes suppression on a destination field cannot fire for this resource, - // which keys its member slice on a composite of feature name, online store name and whether the - // member is streaming. The suppression fires only when the persisted value equals the configured + // which keys its member slice on a composite key defined by materializationSetMemberKey. The + // suppression fires only when the persisted value equals the configured // one, and the persisted-value lookup resolves that composite against the element's field names, // which never match, so it always returns nothing. An unset change.Old is the surrogate for the // persisted value being unreachable; a repoint the config asked for carries that value in from @@ -155,9 +153,9 @@ func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml // DoCreate submits the set and waits for the long-running operation the create returns. The created // set arrives as that operation's response, which Wait reaches by polling GetOperation. func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { - // Each verb takes an idempotency token, and a repeat carrying one already used returns the - // operation the first request created rather than acting again. Minted here so the token the - // server sees is the one this call chose. + // The endpoint takes an idempotency token, and a repeated request carrying one already used + // returns the operation the first request with the same idempotency token created. Minted here + // so the token the server sees is the one this call chose. operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ MaterializationSet: *config, RequestId: uuid.NewString(), @@ -179,9 +177,10 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma return set.MaterializationSetId, set, nil } -// materializationSetChangeFailures names each member a partly applied change could not reach, from -// whichever progress arm the operation's metadata carries. Empty for a creation, which has no -// per-member outcome, and for a failure raised before the server touched a member. +// materializationSetChangeFailures names each materialized feature member a partly applied change +// in the update or delete action could not reach, from whichever progress arm the operation's +// metadata carries. Empty for a creation action, and for a failure (e.g. a validation error) raised +// before the server performed the action. func materializationSetChangeFailures(metadata *ml.MaterializationSetOperationMetadata) []string { if metadata == nil { return nil @@ -202,13 +201,12 @@ func materializationSetChangeFailures(metadata *ml.MaterializationSetOperationMe return failures } -// materializationSetOperationFailure restates a settled-but-failed operation as an API error. The -// generated wait renders the failure's code and message into one string and reads no metadata, so -// the error code is recovered structurally and the per-member progress is appended. +// materializationSetOperationFailure restates a response operation with FAILED status as an API +// error. // -// StatusCode is the 200 the call actually returned, which maps to no sentinel: a member that could -// not be torn down must not reach the destroy path's permission-denied leniency, whose purpose is -// to forgive a resource that is already gone. +// StatusCode is set to 200 because the create, update and delete endpoints always return an +// operation with FAILED status when an error is raised, except for the errors raised before the +// action is executed, such as a validation error on the request. func (r *ResourceMaterializationSet) materializationSetOperationFailure( ctx context.Context, operation materializationSetOperation, From d4b204a9ba0bf86f00538799b37a87ef1cadc34a Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 19:37:39 +0000 Subject: [PATCH 35/47] [FS-2621] Scope the slice-pattern ignore comparison to materialization sets The ignore_remote_changes patterns the test now compares are collected only for this resource, so TestAll compares exactly what it did before for every other one. The pattern-versus-index mismatch is general and jobs, apps and database_instances carry slice-interior rules too, but widening their comparison is a change that should stand on its own. Co-authored-by: Isaac --- bundle/direct/dresources/all_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index b48b1546f35..2b595f331ac 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -977,7 +977,7 @@ type testIgnoreFilter struct { // It also ignores fields that exist in StateType but not in RemoteType, because // those are automatically suppressed by the planner (reason: missing_in_remote) // and RemapState cannot populate them from remote state. -func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { +func newTestIgnoreFilter(adapter *Adapter, group string) *testIgnoreFilter { ignoreFields := make(map[string]bool) var ignorePatterns []*structpath.PatternNode for _, cfg := range []*ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { @@ -986,7 +986,9 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { } for _, p := range cfg.IgnoreRemoteChanges { ignoreFields[p.Field.String()] = true - ignorePatterns = append(ignorePatterns, p.Field) + if group == "materialization_sets" { + ignorePatterns = append(ignorePatterns, p.Field) + } } } // Auto-include fields present in StateType but absent from RemoteType. @@ -1023,8 +1025,10 @@ func (f *testIgnoreFilter) shouldIgnore(path string) bool { return true } - // A rule inside a slice is written as a pattern (foo[*].bar) while a walked path is indexed - // (foo[0].bar), so the lookups above never match one. Compare with the pattern matcher. + // Pattern rules are collected for materialization_sets alone, so this loop is empty for every + // other resource. A rule inside a slice (foo[*].bar) never matches the indexed path a walk + // produces (foo[0].bar); jobs, apps and database_instances carry such rules too, so widening + // the comparison to them is its own change. parsed, err := structpath.ParsePath(path) if err != nil { return false @@ -1125,7 +1129,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NotNil(t, remappedState) // Create filter for fields that should be ignored based on ignore_remote_changes config. - ignoreFilter := newTestIgnoreFilter(adapter) + ignoreFilter := newTestIgnoreFilter(adapter, group) if remoteStateFromCreate != nil { remappedRemoteStateFromCreate, err := adapter.RemapState(remoteStateFromCreate) From 1f20d238b51254abf40a2674beb602528d86208d Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 19:48:32 +0000 Subject: [PATCH 36/47] [FS-2621] Accept day and hour units in a feature's duration fields A reviewer asked whether window_duration can be written in days or hours. It cannot: these are google.protobuf.Duration fields, whose JSON form is fixed at a decimal count of seconds, so a bundle author hand-writes 604800s for a week while the notebook client takes a timedelta. The window types now declare their durations as strings and shadow the same-named fields of the API types they embed, so the dyn-to-typed conversion never hands a duration to protojson - no mutator could repair that, because the conversion runs per file at load and again on entry to every mutator. featureState converts them in the adapter, trying the protobuf form first so a value that deploys today is returned unchanged: a feature's function is immutable, so a rewritten duration would recreate it rather than update it. PrepareInputConfig reports an unparseable value, which PrepareState cannot - its signature carries no error, and the calling convention matches return counts against IResource exactly. Nine fields the API types as Duration and five it types as a plain string; declaring all of them as strings makes that split irrelevant. Verified in the prototype worktree, which carries the local SDK replace. All fourteen fields convert, a variable holding 30m reaches the wire as 1800s, and rewriting 7d as 168h plans nothing. The guard fails when the API gains a duration field the config does not re-type, proven by adding one to ml.RollingWindow. Co-authored-by: Isaac --- .../features/duration/databricks.yml | 83 ++++ .../resources/features/duration/out.test.toml | 2 + .../resources/features/duration/output.txt | 207 +++++++++ .../bundle/resources/features/duration/script | 16 + bundle/config/resources/feature.go | 138 ++++++ bundle/direct/dresources/feature.go | 14 +- bundle/direct/dresources/feature_duration.go | 367 ++++++++++++++++ .../dresources/feature_duration_test.go | 400 ++++++++++++++++++ bundle/direct/dresources/serialize_test.go | 8 +- 9 files changed, 1233 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/resources/features/duration/databricks.yml create mode 100644 acceptance/bundle/resources/features/duration/out.test.toml create mode 100644 acceptance/bundle/resources/features/duration/output.txt create mode 100644 acceptance/bundle/resources/features/duration/script create mode 100644 bundle/direct/dresources/feature_duration.go create mode 100644 bundle/direct/dresources/feature_duration_test.go diff --git a/acceptance/bundle/resources/features/duration/databricks.yml b/acceptance/bundle/resources/features/duration/databricks.yml new file mode 100644 index 00000000000..84d3654993d --- /dev/null +++ b/acceptance/bundle/resources/features/duration/databricks.yml @@ -0,0 +1,83 @@ +bundle: + name: test-bundle + +variables: + settling: + default: 30m + +resources: + features: + rollingfeat: + full_name: main.myschema.rollingfeat + source: + delta_table_source: + full_name: main.myschema.mysource + lateness: + settling_delay: ${var.settling} + function: + aggregation_function: + count_function: + input: amount + time_window: + rolling: + window_duration: 7d + delay: 1h + entities: + - name: account_id + timeseries_column: + name: event_ts + + slidingfeat: + full_name: main.myschema.slidingfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 1w + slide_duration: 1d + offset: 22h + delay: 15m + entities: + - name: account_id + timeseries_column: + name: event_ts + + sawtoothfeat: + full_name: main.myschema.sawtoothfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sawtooth: + window_duration: 3d15m + entities: + - name: account_id + timeseries_column: + name: event_ts + + tumblingfeat: + full_name: main.myschema.tumblingfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + tumbling: + window_duration: 1d + offset: 6h + entities: + - name: account_id + timeseries_column: + name: event_ts diff --git a/acceptance/bundle/resources/features/duration/out.test.toml b/acceptance/bundle/resources/features/duration/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/features/duration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/features/duration/output.txt b/acceptance/bundle/resources/features/duration/output.txt new file mode 100644 index 00000000000..4ae314138b3 --- /dev/null +++ b/acceptance/bundle/resources/features/duration/output.txt @@ -0,0 +1,207 @@ + +=== A bundle may write a duration with week, day, hour and minute units +>>> [CLI] bundle summary -o json +{ + "rollingfeat": { + "rolling": { + "delay": "1h", + "window_duration": "7d" + } + }, + "sawtoothfeat": { + "sawtooth": { + "window_duration": "3d15m" + } + }, + "slidingfeat": { + "sliding": { + "delay": "15m", + "offset": "22h", + "slide_duration": "1d", + "window_duration": "1w" + } + }, + "tumblingfeat": { + "tumbling": { + "offset": "6h", + "window_duration": "1d" + } + } +} + +=== The create requests carry the seconds form the API accepts +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.rollingfeat +Created features.sawtoothfeat +Created features.slidingfeat +Created features.tumblingfeat +Files: 1 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --sort //feature-engineering +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.rollingfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "rolling": { + "delay": "3600s", + "window_duration": "604800s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + }, + "lateness": { + "settling_delay": "1800s" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.sawtoothfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "sawtooth": { + "window_duration": "260100s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.slidingfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "sliding": { + "delay": "900s", + "offset": "79200s", + "slide_duration": "86400s", + "window_duration": "604800s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.tumblingfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "tumbling": { + "offset": "21600s", + "window_duration": "86400s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} + +=== Rewriting a duration in a different unit of the same length plans nothing +>>> update_file.py databricks.yml window_duration: 7d window_duration: 168h + +>>> [CLI] bundle plan -o json +{ + "resources.features.rollingfeat": "skip", + "resources.features.sawtoothfeat": "skip", + "resources.features.slidingfeat": "skip", + "resources.features.tumblingfeat": "skip" +} + +=== An unparseable duration names the field it is on +>>> [CLI] bundle plan --var=settling=7 fortnights +Error: reading config: cannot plan resources.features.rollingfeat: source: lateness: settling_delay: invalid duration "7 fortnights": time: unknown unit " fortnights" in duration "7 fortnights"; use e.g. 604800s, 168h, 7d or 3d15m + + +Exit code: 1 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.rollingfeat + delete resources.features.sawtoothfeat + delete resources.features.slidingfeat + delete resources.features.tumblingfeat + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/features/duration/script b/acceptance/bundle/resources/features/duration/script new file mode 100644 index 00000000000..affe5dd4e9b --- /dev/null +++ b/acceptance/bundle/resources/features/duration/script @@ -0,0 +1,16 @@ +title "A bundle may write a duration with week, day, hour and minute units" +trace $CLI bundle summary -o json | jq -S '.resources.features | map_values(.function.aggregation_function.time_window)' + +title "The create requests carry the seconds form the API accepts" +trace $CLI bundle deploy +trace print_requests.py --sort //feature-engineering + +title "Rewriting a duration in a different unit of the same length plans nothing" +trace update_file.py databricks.yml "window_duration: 7d" "window_duration: 168h" +trace $CLI bundle plan -o json | jq '.plan | with_entries(select(.key | startswith("resources.features"))) | map_values(.action)' + +title "An unparseable duration names the field it is on" +errcode trace $CLI bundle plan --var="settling=7 fortnights" + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt out.plan.json diff --git a/bundle/config/resources/feature.go b/bundle/config/resources/feature.go index 05d27624a2e..be0140e14b1 100644 --- a/bundle/config/resources/feature.go +++ b/bundle/config/resources/feature.go @@ -17,6 +17,144 @@ type Feature struct { BaseResource ID string `json:"id,omitempty" bundle:"readonly"` ml.Feature + + // ========================================= + // === overrides over [ml.Feature] ========= + // ========================================= + + Function FeatureFunction `json:"function"` + Source FeatureDataSource `json:"source"` + TimeWindow *FeatureTimeWindow `json:"time_window,omitempty"` +} + +// The types below re-type the API's duration fields as strings so a bundle can write a duration +// with day and hour units; featureState converts them before deploy. Each embeds the API type and +// shadows only the fields it re-types, so every other field resolves through the embedding. + +type FeatureFunction struct { + ml.Function + AggregationFunction *FeatureAggregationFunction `json:"aggregation_function,omitempty"` +} + +type FeatureAggregationFunction struct { + ml.AggregationFunction + TimeWindow *FeatureTimeWindow `json:"time_window,omitempty"` +} + +type FeatureTimeWindow struct { + ml.TimeWindow + Rolling *FeatureRollingWindow `json:"rolling,omitempty"` + Sawtooth *FeatureSawtoothWindow `json:"sawtooth,omitempty"` + Sliding *FeatureSlidingWindow `json:"sliding,omitempty"` + Tumbling *FeatureTumblingWindow `json:"tumbling,omitempty"` +} + +type FeatureRollingWindow struct { + ml.RollingWindow + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type FeatureSawtoothWindow struct { + ml.SawtoothWindow + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type FeatureSlidingWindow struct { + ml.SlidingWindow + Delay string `json:"delay,omitempty"` + Offset string `json:"offset,omitempty"` +} + +type FeatureTumblingWindow struct { + ml.TumblingWindow + Delay string `json:"delay,omitempty"` + Offset string `json:"offset,omitempty"` +} + +type FeatureDataSource struct { + ml.DataSource + Lateness *FeatureSourceLateness `json:"lateness,omitempty"` +} + +type FeatureSourceLateness struct { + ml.SourceLateness + SettlingDelay string `json:"settling_delay,omitempty"` +} + +// Each wrapper declares its own JSON methods, overriding the ones its embedded API type promotes. + +func (c *FeatureFunction) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureFunction) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureAggregationFunction) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureAggregationFunction) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureTimeWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureTimeWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureRollingWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureRollingWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureSawtoothWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureSawtoothWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureSlidingWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureSlidingWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureTumblingWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureTumblingWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureDataSource) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureDataSource) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *FeatureSourceLateness) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c FeatureSourceLateness) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) } func (f *Feature) UnmarshalJSON(b []byte) error { diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 8d95e3d0eac..b10f78d2c0d 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -5,6 +5,7 @@ import ( "slices" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -20,7 +21,18 @@ func (*ResourceFeature) New(client *databricks.WorkspaceClient) *ResourceFeature } func (*ResourceFeature) PrepareState(input *resources.Feature) *ml.Feature { - return &input.Feature + // An unparseable duration is reported by PrepareInputConfig, which runs first. + state, _ := featureState(input) + return state +} + +// PrepareInputConfig reports a duration featureState cannot parse, which PrepareState cannot: its +// signature carries no error. +func (*ResourceFeature) PrepareInputConfig(input *resources.Feature, _ string) (*structvar.StructVar, error) { + if _, err := featureState(input); err != nil { + return nil, err + } + return &structvar.StructVar{Value: input, Refs: nil}, nil } func (r *ResourceFeature) DoRead(ctx context.Context, id string) (*ml.Feature, error) { diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go new file mode 100644 index 00000000000..01ba2096832 --- /dev/null +++ b/bundle/direct/dresources/feature_duration.go @@ -0,0 +1,367 @@ +package dresources + +import ( + "encoding/json" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn/dynvar" + sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// durationComponent matches one leading "" component of a duration string, including +// the day (d) and week (w) units time.ParseDuration lacks. +var durationComponent = regexp.MustCompile(`^([0-9]+(?:\.[0-9]+)?)(ns|us|µs|ms|s|m|h|d|w)`) + +const ( + hoursPerDay = 24 + hoursPerWeek = 24 * 7 + // maxDurationHours is the largest whole-hour count representable as a time.Duration (an int64 + // nanosecond count). + maxDurationHours = int64(math.MaxInt64) / int64(time.Hour) +) + +// duration parses a duration written in the API's protobuf form ("604800s"), Go form ("168h"), or +// with the day and week units Go lacks ("7d", "3d15m"). +func duration(s string) (*sdkduration.Duration, error) { + if parsed, ok := protobufDuration(s); ok { + return parsed, nil + } + d, err := parseExtendedDuration(s) + if err != nil { + return nil, fmt.Errorf("invalid duration %q: %w; use e.g. 604800s, 168h, 7d or 3d15m", s, err) + } + return sdkduration.New(d), nil +} + +// canonicalDuration returns s in the API's protobuf form, returning s unchanged when it already is. +func canonicalDuration(s string) (string, error) { + if _, ok := protobufDuration(s); ok { + return s, nil + } + d, err := parseExtendedDuration(s) + if err != nil { + return "", fmt.Errorf("invalid duration %q: %w; use e.g. 604800s, 168h, 7d or 3d15m", s, err) + } + marshalled, err := sdkduration.New(d).MarshalJSON() + if err != nil { + return "", err + } + var canonical string + if err := json.Unmarshal(marshalled, &canonical); err != nil { + return "", err + } + return canonical, nil +} + +// protobufDuration reports whether s is already a protobuf duration, returning the parsed value. +func protobufDuration(s string) (*sdkduration.Duration, bool) { + quoted, err := json.Marshal(s) + if err != nil { + return nil, false + } + var parsed sdkduration.Duration + if err := parsed.UnmarshalJSON(quoted); err != nil { + return nil, false + } + return &parsed, true +} + +// parseExtendedDuration parses a Go duration string extended with day (d, = 24h) and week (w, = +// 168h) units. Components in those units are summed here; all other components (h and below) are +// handed to time.ParseDuration unchanged, which keeps its exact integer handling for the common +// forms. +func parseExtendedDuration(s string) (time.Duration, error) { + // Fast path: no day/week unit means time.ParseDuration handles it all. No other supported unit + // contains 'd' or 'w', so this check is exact. + if !strings.ContainsAny(s, "dw") { + return time.ParseDuration(s) + } + + rest := s + neg := false + switch { + case strings.HasPrefix(rest, "-"): + neg, rest = true, rest[1:] + case strings.HasPrefix(rest, "+"): + rest = rest[1:] + } + + var extra time.Duration // accumulated day/week components + var residual strings.Builder // remaining components, parsed by time.ParseDuration + for rest != "" { + m := durationComponent.FindStringSubmatch(rest) + if m == nil { + return 0, fmt.Errorf("unrecognized duration component %q", rest) + } + switch m[2] { + case "d", "w": + hoursPerUnit := int64(hoursPerDay) + if m[2] == "w" { + hoursPerUnit = hoursPerWeek + } + if strings.Contains(m[1], ".") { + f, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, err + } + ns := f * float64(hoursPerUnit) * float64(time.Hour) + if ns > float64(math.MaxInt64) { + return 0, fmt.Errorf("duration component %s%s overflows", m[1], m[2]) + } + extra += time.Duration(ns) + } else { + n, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, err + } + if n > maxDurationHours/hoursPerUnit { + return 0, fmt.Errorf("duration component %s%s overflows", m[1], m[2]) + } + extra += time.Duration(n*hoursPerUnit) * time.Hour + } + default: + residual.WriteString(m[1]) + residual.WriteString(m[2]) + } + rest = rest[len(m[0]):] + } + + total := extra + if residual.Len() > 0 { + d, err := time.ParseDuration(residual.String()) + if err != nil { + return 0, err + } + total += d + } + if neg { + total = -total + } + return total, nil +} + +// optionalDuration converts a configured duration into the API type, leaving an unset value and an +// unresolved reference alone. +func optionalDuration(s string) (*sdkduration.Duration, error) { + if s == "" || dynvar.IsPureVariableReference(s) { + return nil, nil + } + return duration(s) +} + +// optionalCanonicalDuration canonicalizes a duration the API types as a string, under the same two +// exemptions as optionalDuration. +func optionalCanonicalDuration(s string) (string, error) { + if s == "" || dynvar.IsPureVariableReference(s) { + return s, nil + } + return canonicalDuration(s) +} + +// featureState converts the bundle config for a feature into the API type, parsing every duration. +// The config declares durations as strings; see the override note in +// bundle/config/resources/feature.go. +func featureState(input *resources.Feature) (*ml.Feature, error) { + state := input.Feature + + function, err := featureFunctionState(&input.Function) + if err != nil { + return nil, fmt.Errorf("function: %w", err) + } + state.Function = *function + + source, err := featureDataSourceState(&input.Source) + if err != nil { + return nil, fmt.Errorf("source: %w", err) + } + state.Source = *source + + // The deprecated top-level time_window is converted on the same terms as the one under the + // aggregation function. + state.TimeWindow, err = featureTimeWindowState(input.TimeWindow) + if err != nil { + return nil, fmt.Errorf("time_window: %w", err) + } + + return &state, nil +} + +func featureFunctionState(input *resources.FeatureFunction) (*ml.Function, error) { + state := input.Function + + aggregation, err := featureAggregationFunctionState(input.AggregationFunction) + if err != nil { + return nil, fmt.Errorf("aggregation_function: %w", err) + } + state.AggregationFunction = aggregation + + return &state, nil +} + +func featureAggregationFunctionState(input *resources.FeatureAggregationFunction) (*ml.AggregationFunction, error) { + if input == nil { + return nil, nil + } + state := input.AggregationFunction + + timeWindow, err := featureTimeWindowState(input.TimeWindow) + if err != nil { + return nil, fmt.Errorf("time_window: %w", err) + } + state.TimeWindow = timeWindow + + return &state, nil +} + +func featureTimeWindowState(input *resources.FeatureTimeWindow) (*ml.TimeWindow, error) { + if input == nil { + return nil, nil + } + state := input.TimeWindow + + var err error + if state.Rolling, err = rollingWindowState(input.Rolling); err != nil { + return nil, fmt.Errorf("rolling: %w", err) + } + if state.Sawtooth, err = sawtoothWindowState(input.Sawtooth); err != nil { + return nil, fmt.Errorf("sawtooth: %w", err) + } + if state.Sliding, err = slidingWindowState(input.Sliding); err != nil { + return nil, fmt.Errorf("sliding: %w", err) + } + if state.Tumbling, err = tumblingWindowState(input.Tumbling); err != nil { + return nil, fmt.Errorf("tumbling: %w", err) + } + if state.Continuous, err = continuousWindowState(state.Continuous); err != nil { + return nil, fmt.Errorf("continuous: %w", err) + } + + return &state, nil +} + +func rollingWindowState(input *resources.FeatureRollingWindow) (*ml.RollingWindow, error) { + if input == nil { + return nil, nil + } + state := input.RollingWindow + + var err error + if state.Delay, err = optionalDuration(input.Delay); err != nil { + return nil, fmt.Errorf("delay: %w", err) + } + if state.WindowDuration, err = optionalDuration(input.WindowDuration); err != nil { + return nil, fmt.Errorf("window_duration: %w", err) + } + + return &state, nil +} + +func sawtoothWindowState(input *resources.FeatureSawtoothWindow) (*ml.SawtoothWindow, error) { + if input == nil { + return nil, nil + } + state := input.SawtoothWindow + + var err error + if state.Delay, err = optionalDuration(input.Delay); err != nil { + return nil, fmt.Errorf("delay: %w", err) + } + if state.WindowDuration, err = optionalDuration(input.WindowDuration); err != nil { + return nil, fmt.Errorf("window_duration: %w", err) + } + + return &state, nil +} + +func slidingWindowState(input *resources.FeatureSlidingWindow) (*ml.SlidingWindow, error) { + if input == nil { + return nil, nil + } + state := input.SlidingWindow + + var err error + if state.Delay, err = optionalDuration(input.Delay); err != nil { + return nil, fmt.Errorf("delay: %w", err) + } + if state.Offset, err = optionalDuration(input.Offset); err != nil { + return nil, fmt.Errorf("offset: %w", err) + } + if state.WindowDuration, err = optionalCanonicalDuration(state.WindowDuration); err != nil { + return nil, fmt.Errorf("window_duration: %w", err) + } + if state.SlideDuration, err = optionalCanonicalDuration(state.SlideDuration); err != nil { + return nil, fmt.Errorf("slide_duration: %w", err) + } + + return &state, nil +} + +func tumblingWindowState(input *resources.FeatureTumblingWindow) (*ml.TumblingWindow, error) { + if input == nil { + return nil, nil + } + state := input.TumblingWindow + + var err error + if state.Delay, err = optionalDuration(input.Delay); err != nil { + return nil, fmt.Errorf("delay: %w", err) + } + if state.Offset, err = optionalDuration(input.Offset); err != nil { + return nil, fmt.Errorf("offset: %w", err) + } + if state.WindowDuration, err = optionalCanonicalDuration(state.WindowDuration); err != nil { + return nil, fmt.Errorf("window_duration: %w", err) + } + + return &state, nil +} + +func continuousWindowState(input *ml.ContinuousWindow) (*ml.ContinuousWindow, error) { + if input == nil { + return nil, nil + } + state := *input + + var err error + if state.Offset, err = optionalCanonicalDuration(state.Offset); err != nil { + return nil, fmt.Errorf("offset: %w", err) + } + if state.WindowDuration, err = optionalCanonicalDuration(state.WindowDuration); err != nil { + return nil, fmt.Errorf("window_duration: %w", err) + } + + return &state, nil +} + +func featureDataSourceState(input *resources.FeatureDataSource) (*ml.DataSource, error) { + state := input.DataSource + + lateness, err := featureSourceLatenessState(input.Lateness) + if err != nil { + return nil, fmt.Errorf("lateness: %w", err) + } + state.Lateness = lateness + + return &state, nil +} + +func featureSourceLatenessState(input *resources.FeatureSourceLateness) (*ml.SourceLateness, error) { + if input == nil { + return nil, nil + } + state := input.SourceLateness + + var err error + if state.SettlingDelay, err = optionalDuration(input.SettlingDelay); err != nil { + return nil, fmt.Errorf("settling_delay: %w", err) + } + + return &state, nil +} diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go new file mode 100644 index 00000000000..e8244777920 --- /dev/null +++ b/bundle/direct/dresources/feature_duration_test.go @@ -0,0 +1,400 @@ +package dresources + +import ( + "reflect" + "strings" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structtag" + sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" + sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseExtendedDuration(t *testing.T) { + for _, tc := range []struct { + input string + want string // canonical protobuf form + wantErr bool + }{ + {input: "604800s", want: "604800s"}, + {input: "168h", want: "604800s"}, + {input: "7d", want: "604800s"}, + {input: "1w", want: "604800s"}, + {input: "3d15m", want: "260100s"}, + {input: "1.5d", want: "129600s"}, + {input: "0.5s", want: "0.5s"}, + {input: "", wantErr: true}, + {input: "abc", wantErr: true}, + {input: "7x", wantErr: true}, + {input: "7dd", wantErr: true}, + {input: "100000000000w", wantErr: true}, + } { + t.Run(tc.input, func(t *testing.T) { + got, err := canonicalDuration(tc.input) + if tc.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +// TestCanonicalDurationPreservesProtobufForm pins the no-churn guarantee: a value the API already +// accepts comes back byte-for-byte, so an existing bundle's state does not move. +func TestCanonicalDurationPreservesProtobufForm(t *testing.T) { + for _, input := range []string{"604800s", "0.5s", "0.500s", "1.000000001s", "0s"} { + got, err := canonicalDuration(input) + require.NoError(t, err, input) + assert.Equal(t, input, got) + } +} + +func seconds(t *testing.T, d *sdkduration.Duration) float64 { + t.Helper() + require.NotNil(t, d) + return d.AsDuration().Seconds() +} + +func TestFeatureStateConvertsEveryWindow(t *testing.T) { + state, err := featureState(featureWithEveryDuration()) + require.NoError(t, err) + + window := state.Function.AggregationFunction.TimeWindow + assert.Equal(t, 86400.0, seconds(t, window.Rolling.WindowDuration)) + assert.Equal(t, 86400.0, seconds(t, window.Rolling.Delay)) + assert.Equal(t, 86400.0, seconds(t, window.Sawtooth.WindowDuration)) + assert.Equal(t, 86400.0, seconds(t, window.Sliding.Offset)) + assert.Equal(t, "86400s", window.Sliding.WindowDuration) + assert.Equal(t, "86400s", window.Sliding.SlideDuration) + assert.Equal(t, "86400s", window.Tumbling.WindowDuration) + assert.Equal(t, "86400s", window.Continuous.WindowDuration) + assert.Equal(t, 86400.0, seconds(t, state.Source.Lateness.SettlingDelay)) + + // The deprecated top-level time_window is converted on the same terms. + assert.Equal(t, 86400.0, seconds(t, state.TimeWindow.Rolling.WindowDuration)) +} + +func TestFeatureStateAcceptsCompoundAndProtobufForms(t *testing.T) { + input := &resources.Feature{} + input.Function.AggregationFunction = &resources.FeatureAggregationFunction{ + TimeWindow: &resources.FeatureTimeWindow{ + Sawtooth: &resources.FeatureSawtoothWindow{WindowDuration: "3d15m"}, + Sliding: &resources.FeatureSlidingWindow{}, + }, + } + input.Function.AggregationFunction.TimeWindow.Sliding.SlideDuration = "604800s" + + state, err := featureState(input) + require.NoError(t, err) + window := state.Function.AggregationFunction.TimeWindow + assert.Equal(t, 260100.0, seconds(t, window.Sawtooth.WindowDuration)) + assert.Equal(t, "604800s", window.Sliding.SlideDuration) +} + +func TestFeatureStateLeavesUnsetAndVariableReferences(t *testing.T) { + input := &resources.Feature{} + input.Function.AggregationFunction = &resources.FeatureAggregationFunction{ + TimeWindow: &resources.FeatureTimeWindow{ + Rolling: &resources.FeatureRollingWindow{WindowDuration: "${var.window}"}, + Sliding: &resources.FeatureSlidingWindow{}, + }, + } + + state, err := featureState(input) + require.NoError(t, err) + window := state.Function.AggregationFunction.TimeWindow + assert.Nil(t, window.Rolling.WindowDuration, "an unresolved reference is left alone") + assert.Nil(t, window.Rolling.Delay, "an unset duration stays unset") + assert.Empty(t, window.Sliding.WindowDuration) +} + +func TestFeatureStateWithoutWindows(t *testing.T) { + state, err := featureState(&resources.Feature{}) + require.NoError(t, err) + assert.Nil(t, state.Function.AggregationFunction) + assert.Nil(t, state.Source.Lateness) + assert.Nil(t, state.TimeWindow) +} + +func TestFeaturePrepareInputConfigReportsBadDuration(t *testing.T) { + input := &resources.Feature{} + input.Function.AggregationFunction = &resources.FeatureAggregationFunction{ + TimeWindow: &resources.FeatureTimeWindow{ + Rolling: &resources.FeatureRollingWindow{WindowDuration: "7 fortnights"}, + }, + } + + _, err := (*ResourceFeature)(nil).PrepareInputConfig(input, "resources.features.f") + require.Error(t, err) + assert.Contains(t, err.Error(), "function: aggregation_function: time_window: rolling: window_duration:") + assert.Contains(t, err.Error(), `"7 fortnights"`) +} + +func TestFeaturePrepareInputConfigPassesConfigThrough(t *testing.T) { + input := featureWithEveryDuration() + got, err := (*ResourceFeature)(nil).PrepareInputConfig(input, "resources.features.f") + require.NoError(t, err) + assert.Same(t, input, got.Value) + assert.Nil(t, got.Refs) +} + +// ---- the guard ---- + +// durationNamed reports whether a string-typed field holds a duration. The API types five of these +// as plain strings, and neither the Go type nor the generated schema says so, so the name is the +// only signal. +func durationNamed(jsonName string) bool { + leaf := jsonName + if i := strings.LastIndex(leaf, "."); i >= 0 { + leaf = leaf[i+1:] + } + // freshness_target is the one duration-shaped name that follows no convention, so it is named. + return strings.HasSuffix(leaf, "_duration") || strings.HasSuffix(leaf, "delay") || + leaf == "offset" || leaf == "freshness_target" +} + +var durationType = reflect.TypeFor[sdkduration.Duration]() + +func terminalStruct(typ reflect.Type) bool { + return typ == durationType || typ == reflect.TypeFor[sdktime.Time]() || + typ == reflect.TypeFor[sdkfieldmask.FieldMask]() +} + +// flattenedFields returns typ's fields the way encoding/json sees them: breadth-first from the +// outer struct inward, with a JSON name declared at a shallower depth hiding a deeper one, matching +// buildStructInfo in libs/dyn/convert/struct_info.go. +func flattenedFields(typ reflect.Type) []reflect.StructField { + var out []reflect.StructField + seen := map[string]bool{} + queue := []reflect.Type{typ} + + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + + var embedded []reflect.Type + for i := range cur.NumField() { + field := cur.Field(i) + name := structtag.JSONTag(field.Tag.Get("json")).Name() + + if field.Anonymous && name == "" { + inner := field.Type + for inner.Kind() == reflect.Pointer { + inner = inner.Elem() + } + if inner.Kind() == reflect.Struct { + embedded = append(embedded, inner) + continue + } + } + if !field.IsExported() || name == "" || name == "-" || seen[name] { + continue + } + seen[name] = true + out = append(out, field) + } + queue = append(queue, embedded...) + } + return out +} + +// jsonLeaves maps the JSON path of every leaf reachable from typ to the leaf's Go type. +func jsonLeaves(typ reflect.Type) map[string]reflect.Type { + out := map[string]reflect.Type{} + collectJSONLeaves(typ, "", map[reflect.Type]bool{}, out) + return out +} + +func collectJSONLeaves(typ reflect.Type, prefix string, onPath map[reflect.Type]bool, out map[string]reflect.Type) { + for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice { + typ = typ.Elem() + } + if typ.Kind() != reflect.Struct || onPath[typ] { + return + } + onPath[typ] = true + defer delete(onPath, typ) + + for _, field := range flattenedFields(typ) { + path := structtag.JSONTag(field.Tag.Get("json")).Name() + if prefix != "" { + path = prefix + "." + path + } + + inner := field.Type + for inner.Kind() == reflect.Pointer || inner.Kind() == reflect.Slice { + inner = inner.Elem() + } + if inner.Kind() == reflect.Struct && !terminalStruct(inner) { + collectJSONLeaves(inner, path, onPath, out) + continue + } + out[path] = field.Type + } +} + +func isDurationTyped(typ reflect.Type) bool { + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + return typ == durationType +} + +// TestFeatureDurationFieldsAreRetyped fails when the API gains a duration field the bundle config +// does not re-type as a string, including one on a window kind that does not exist yet. +func TestFeatureDurationFieldsAreRetyped(t *testing.T) { + sdk := jsonLeaves(reflect.TypeFor[ml.Feature]()) + config := jsonLeaves(reflect.TypeFor[resources.Feature]()) + + found := 0 + for path, typ := range sdk { + if !isDurationTyped(typ) { + continue + } + found++ + configType, ok := config[path] + require.Truef(t, ok, "ml.Feature has duration field %q, missing from resources.Feature", path) + assert.Equalf(t, reflect.String, configType.Kind(), + "%q must be re-typed as a string in resources.Feature so a bundle can write 7d; "+ + "add a wrapper in bundle/config/resources/feature.go and convert it in featureState", + path) + } + assert.Positive(t, found, "walked no duration fields, the walker is broken") +} + +// TestFeatureShadowsAreNotStale fails when a re-typed field no longer matches a field of the API +// type, which is what an upstream rename leaves behind. +func TestFeatureShadowsAreNotStale(t *testing.T) { + sdkFields := map[string]bool{} + for _, field := range flattenedFields(reflect.TypeFor[ml.Feature]()) { + sdkFields[structtag.JSONTag(field.Tag.Get("json")).Name()] = true + } + + // The subtrees resources.Feature re-types are those whose top-level name it redeclares. + shadowed := map[string]bool{} + configType := reflect.TypeFor[resources.Feature]() + for i := range configType.NumField() { + field := configType.Field(i) + name := structtag.JSONTag(field.Tag.Get("json")).Name() + if !field.Anonymous && sdkFields[name] { + shadowed[name] = true + } + } + require.NotEmpty(t, shadowed, "found no re-typed subtree, the walker is broken") + + sdk := jsonLeaves(reflect.TypeFor[ml.Feature]()) + for path := range jsonLeaves(configType) { + if !shadowed[strings.SplitN(path, ".", 2)[0]] { + continue + } + _, ok := sdk[path] + assert.Truef(t, ok, "resources.Feature declares %q but ml.Feature does not; a re-typed field is stale", path) + } +} + +// TestFeatureStateConvertsEveryDurationField is the coverage guard: every duration field the API +// declares must come out of featureState holding the configured value. A field that is re-typed but +// never converted arrives nil and fails here. +func TestFeatureStateConvertsEveryDurationField(t *testing.T) { + state, err := featureState(featureWithEveryDuration()) + require.NoError(t, err) + + checked := 0 + for path, typ := range jsonLeaves(reflect.TypeFor[ml.Feature]()) { + if !isDurationTyped(typ) && !(typ.Kind() == reflect.String && durationNamed(path)) { + continue + } + checked++ + value := leafByPath(t, reflect.ValueOf(state), path) + require.Truef(t, value.IsValid(), "%q is unreachable in the converted state", path) + + if isDurationTyped(typ) { + require.Falsef(t, value.IsNil(), + "%q came out unset: add it to featureState and to featureWithEveryDuration", path) + assert.Equalf(t, 86400.0, value.Interface().(*sdkduration.Duration).AsDuration().Seconds(), path) + continue + } + assert.Equalf(t, "86400s", value.String(), + "%q came out unconverted: add it to featureState and to featureWithEveryDuration", path) + } + assert.Positive(t, checked, "walked no duration fields, the walker is broken") +} + +// TestMaterializationSetDurationFieldsAreKnown fails when the API gives a materialization set a +// second duration-shaped field. freshness_target has no reader in the service, so the resource +// deliberately converts nothing; a new field forces that decision to be revisited. +func TestMaterializationSetDurationFieldsAreKnown(t *testing.T) { + var found []string + for path, typ := range jsonLeaves(reflect.TypeFor[ml.MaterializationSet]()) { + if isDurationTyped(typ) || (typ.Kind() == reflect.String && durationNamed(path)) { + found = append(found, path) + } + } + assert.ElementsMatch(t, []string{"materialized_features.streaming_mode.freshness_target"}, found) +} + +// leafByPath resolves a dotted JSON path against a struct value, following the same shadowing rule +// as flattenedFields. +func leafByPath(t *testing.T, v reflect.Value, path string) reflect.Value { + t.Helper() + for _, segment := range strings.Split(path, ".") { + for v.Kind() == reflect.Pointer { + if v.IsNil() { + return reflect.Value{} + } + v = v.Elem() + } + if v.Kind() != reflect.Struct { + return reflect.Value{} + } + var next reflect.Value + for _, field := range flattenedFields(v.Type()) { + if structtag.JSONTag(field.Tag.Get("json")).Name() != segment { + continue + } + next = v.FieldByIndex(field.Index) + break + } + if !next.IsValid() { + return reflect.Value{} + } + v = next + } + return v +} + +// featureWithEveryDuration sets every duration field a feature has to one day, so the coverage +// guard can assert each one survives conversion. +func featureWithEveryDuration() *resources.Feature { + const day = "1d" + + window := func() *resources.FeatureTimeWindow { + w := &resources.FeatureTimeWindow{ + Rolling: &resources.FeatureRollingWindow{Delay: day, WindowDuration: day}, + Sawtooth: &resources.FeatureSawtoothWindow{Delay: day, WindowDuration: day}, + Sliding: &resources.FeatureSlidingWindow{Delay: day, Offset: day}, + Tumbling: &resources.FeatureTumblingWindow{Delay: day, Offset: day}, + } + w.Sliding.WindowDuration = day + w.Sliding.SlideDuration = day + w.Tumbling.WindowDuration = day + w.Continuous = &ml.ContinuousWindow{Offset: day, WindowDuration: day} + return w + } + + input := &resources.Feature{ + Function: resources.FeatureFunction{ + AggregationFunction: &resources.FeatureAggregationFunction{TimeWindow: window()}, + }, + Source: resources.FeatureDataSource{Lateness: &resources.FeatureSourceLateness{SettlingDelay: day}}, + TimeWindow: window(), + } + return input +} diff --git a/bundle/direct/dresources/serialize_test.go b/bundle/direct/dresources/serialize_test.go index 76078460193..a7827216a6f 100644 --- a/bundle/direct/dresources/serialize_test.go +++ b/bundle/direct/dresources/serialize_test.go @@ -145,8 +145,14 @@ func TestRoundtripAllFieldsRemoteType(t *testing.T) { // marshal.Unmarshal hands the whole payload to the embedded member, so the shadowed // string comes back holding the raw JSON text and reads as a lost field. Its other // fields go unchecked on this surface as a result. +// +// features is exempt for the same reason. Its window types declare their durations as strings and +// deliberately shadow the same-named duration fields of the embedded API types, so a bundle can +// write "7d" (featureState parses it to the protobuf form before deploy). Only the shallower string +// is reachable by that JSON name, so the shadowed duration cannot round-trip, and marshal.Unmarshal +// leaves the embedded members' other fields unset. Its other fields go unchecked here as a result. func TestRoundtripAllFieldsInputConfigType(t *testing.T) { - testRoundtripAllFields(t, "InputConfigType", (*Adapter).InputConfigType, []string{"cluster_policies"}) + testRoundtripAllFields(t, "InputConfigType", (*Adapter).InputConfigType, []string{"cluster_policies", "features"}) } var jsonMarshalerType = reflect.TypeFor[json.Marshaler]() From 9bffa15baefe7bb300e69cf61109e91d7a97499e Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 21:41:13 +0000 Subject: [PATCH 37/47] [FS-2621] Fold the canonical duration helper into its sole caller canonicalDuration duplicated duration's parse and then rendered the result as a string. It now delegates, which also drops its verbatim passthrough: a duration already written with an "s" suffix is re-rendered the way the API renders it, so 0.5s is stored as 0.500s and matches what the server answers with. Also names the postgres function parseExtendedDuration duplicates. Co-authored-by: Isaac --- bundle/direct/dresources/feature_duration.go | 45 +++++++++---------- .../dresources/feature_duration_test.go | 24 ++++++---- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index 01ba2096832..4ce805a587c 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -27,8 +27,8 @@ const ( maxDurationHours = int64(math.MaxInt64) / int64(time.Hour) ) -// duration parses a duration written in the API's protobuf form ("604800s"), Go form ("168h"), or -// with the day and week units Go lacks ("7d", "3d15m"). +// duration parses a duration composed of week, day, hour, minute and second units ("1w", "7d", +// "3d15m", "604800s") into the SDK type, which holds only seconds. func duration(s string) (*sdkduration.Duration, error) { if parsed, ok := protobufDuration(s); ok { return parsed, nil @@ -40,26 +40,6 @@ func duration(s string) (*sdkduration.Duration, error) { return sdkduration.New(d), nil } -// canonicalDuration returns s in the API's protobuf form, returning s unchanged when it already is. -func canonicalDuration(s string) (string, error) { - if _, ok := protobufDuration(s); ok { - return s, nil - } - d, err := parseExtendedDuration(s) - if err != nil { - return "", fmt.Errorf("invalid duration %q: %w; use e.g. 604800s, 168h, 7d or 3d15m", s, err) - } - marshalled, err := sdkduration.New(d).MarshalJSON() - if err != nil { - return "", err - } - var canonical string - if err := json.Unmarshal(marshalled, &canonical); err != nil { - return "", err - } - return canonical, nil -} - // protobufDuration reports whether s is already a protobuf duration, returning the parsed value. func protobufDuration(s string) (*sdkduration.Duration, bool) { quoted, err := json.Marshal(s) @@ -77,6 +57,9 @@ func protobufDuration(s string) (*sdkduration.Duration, bool) { // 168h) units. Components in those units are summed here; all other components (h and below) are // handed to time.ParseDuration unchanged, which keeps its exact integer handling for the common // forms. +// +// Duplicates parseTTLDuration in cmd/workspace/postgres/overrides.go, which serves the postgres +// branch --ttl flag. func parseExtendedDuration(s string) (time.Duration, error) { // Fast path: no day/week unit means time.ParseDuration handles it all. No other supported unit // contains 'd' or 'w', so this check is exact. @@ -156,13 +139,25 @@ func optionalDuration(s string) (*sdkduration.Duration, error) { return duration(s) } -// optionalCanonicalDuration canonicalizes a duration the API types as a string, under the same two -// exemptions as optionalDuration. +// optionalCanonicalDuration renders a duration the API types as a string in the seconds form the +// API answers with, under the same two exemptions as optionalDuration. func optionalCanonicalDuration(s string) (string, error) { if s == "" || dynvar.IsPureVariableReference(s) { return s, nil } - return canonicalDuration(s) + parsed, err := duration(s) + if err != nil { + return "", err + } + marshalled, err := parsed.MarshalJSON() + if err != nil { + return "", err + } + var canonical string + if err := json.Unmarshal(marshalled, &canonical); err != nil { + return "", err + } + return canonical, nil } // featureState converts the bundle config for a feature into the API type, parsing every duration. diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go index e8244777920..564916c0a13 100644 --- a/bundle/direct/dresources/feature_duration_test.go +++ b/bundle/direct/dresources/feature_duration_test.go @@ -27,15 +27,15 @@ func TestParseExtendedDuration(t *testing.T) { {input: "1w", want: "604800s"}, {input: "3d15m", want: "260100s"}, {input: "1.5d", want: "129600s"}, - {input: "0.5s", want: "0.5s"}, - {input: "", wantErr: true}, + {input: "0.5s", want: "0.500s"}, + {input: "", want: ""}, {input: "abc", wantErr: true}, {input: "7x", wantErr: true}, {input: "7dd", wantErr: true}, {input: "100000000000w", wantErr: true}, } { t.Run(tc.input, func(t *testing.T) { - got, err := canonicalDuration(tc.input) + got, err := optionalCanonicalDuration(tc.input) if tc.wantErr { assert.Error(t, err) return @@ -46,13 +46,19 @@ func TestParseExtendedDuration(t *testing.T) { } } -// TestCanonicalDurationPreservesProtobufForm pins the no-churn guarantee: a value the API already -// accepts comes back byte-for-byte, so an existing bundle's state does not move. -func TestCanonicalDurationPreservesProtobufForm(t *testing.T) { - for _, input := range []string{"604800s", "0.5s", "0.500s", "1.000000001s", "0s"} { - got, err := canonicalDuration(input) +// TestOptionalCanonicalDurationRendersProtobufForm pins that a value the API already accepts is +// still rendered the way the API renders it, so state matches what the server answers with. +func TestOptionalCanonicalDurationRendersProtobufForm(t *testing.T) { + for input, want := range map[string]string{ + "604800s": "604800s", + "0.5s": "0.500s", + "0.500s": "0.500s", + "1.000000001s": "1.000000001s", + "0s": "0s", + } { + got, err := optionalCanonicalDuration(input) require.NoError(t, err, input) - assert.Equal(t, input, got) + assert.Equal(t, want, got, input) } } From ae6ef4d9aef9772e495b3d4b09dcc28f3f78a3ad Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 21:41:33 +0000 Subject: [PATCH 38/47] [FS-2621] Write the acceptance durations in mixed units The three fixtures spelled every window in seconds, which exercised only the passthrough. They now cover the week, day, hour and compound forms, each equal to the value it replaces so no golden moves. Co-authored-by: Isaac --- .../bundle/invariant/configs/materialization_set.yml.tmpl | 4 ++-- .../resources/materialization_sets/basic/databricks.yml | 4 ++-- .../resources/materialization_sets/batch/databricks.yml.tmpl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl index 9966eadc384..6549c9af70c 100644 --- a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl @@ -14,8 +14,8 @@ resources: input: value time_window: sliding: - window_duration: 604800s - slide_duration: 86400s + window_duration: 1w + slide_duration: 24h entities: - name: id timeseries_column: diff --git a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml index c347eae6cd1..4473f2068f8 100644 --- a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml +++ b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml @@ -14,8 +14,8 @@ resources: input: amount time_window: sliding: - window_duration: 604800s - slide_duration: 86400s + window_duration: 7d + slide_duration: 1d entities: - name: account_id timeseries_column: diff --git a/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl index 83aebb13548..4d80c657479 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl @@ -14,8 +14,8 @@ resources: input: value time_window: sliding: - window_duration: 604800s - slide_duration: 86400s + window_duration: 6d24h + slide_duration: 23h60m entities: - name: id timeseries_column: From 495a363dadcdf326f3c01b3a0adebe81a33a9b64 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 23 Oct 2026 21:59:56 +0000 Subject: [PATCH 39/47] [FS-2621] Walk a registry of duration fields instead of repeating the conversion Each window repeated the same parse-and-wrap block per duration field, so a field the API adds is converted only if someone remembers to add a block for it. The fields are now declared once as a registry and one loop applies them, keeping the per-field error prefix the goldens record. TestFeatureDurationRegistryIsComplete compares the registry against the duration fields reflection finds on ml.Feature, in both directions, so a new field or a dropped one fails by name. A field whose SDK type flips between a duration and a string fails to compile, because the two registry constructors take different pointer types. Co-authored-by: Isaac --- bundle/direct/dresources/feature_duration.go | 146 +++++++++++++----- .../dresources/feature_duration_test.go | 55 +++++++ 2 files changed, 159 insertions(+), 42 deletions(-) diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index 4ce805a587c..0a56c21deba 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -160,6 +160,98 @@ func optionalCanonicalDuration(s string) (string, error) { return canonical, nil } +// durationConversion is one duration field's conversion: the API field name, which names it in the +// error, and the assignment itself. +type durationConversion struct { + field string + apply func() error +} + +// shadowedDuration parses the string the bundle config shadows an SDK duration field with. +func shadowedDuration(field, source string, dest **sdkduration.Duration) durationConversion { + return durationConversion{field: field, apply: func() error { + parsed, err := optionalDuration(source) + if err != nil { + return err + } + *dest = parsed + return nil + }} +} + +// inPlaceDuration rewrites a duration the SDK types as a string, which the config does not shadow. +func inPlaceDuration(field string, dest *string) durationConversion { + return durationConversion{field: field, apply: func() error { + rewritten, err := optionalCanonicalDuration(*dest) + if err != nil { + return err + } + *dest = rewritten + return nil + }} +} + +// convertDurations applies a registry, naming the field that failed. +// +// A field whose SDK type flips between *sdkduration.Duration and string has to move between +// shadowedDuration and inPlaceDuration, which take different pointer types, so the compiler reports +// it. A duration field the API adds is reported by TestFeatureDurationRegistryIsComplete. +func convertDurations(conversions []durationConversion) error { + for _, conversion := range conversions { + if err := conversion.apply(); err != nil { + return fmt.Errorf("%s: %w", conversion.field, err) + } + } + return nil +} + +// The registry: every duration field the adapter converts, one entry per field. A spec change that +// adds a duration field is one line here. + +func featureSourceLatenessDurations(input *resources.FeatureSourceLateness, state *ml.SourceLateness) []durationConversion { + return []durationConversion{ + shadowedDuration("settling_delay", input.SettlingDelay, &state.SettlingDelay), + } +} + +func rollingWindowDurations(input *resources.FeatureRollingWindow, state *ml.RollingWindow) []durationConversion { + return []durationConversion{ + shadowedDuration("delay", input.Delay, &state.Delay), + shadowedDuration("window_duration", input.WindowDuration, &state.WindowDuration), + } +} + +func sawtoothWindowDurations(input *resources.FeatureSawtoothWindow, state *ml.SawtoothWindow) []durationConversion { + return []durationConversion{ + shadowedDuration("delay", input.Delay, &state.Delay), + shadowedDuration("window_duration", input.WindowDuration, &state.WindowDuration), + } +} + +func slidingWindowDurations(input *resources.FeatureSlidingWindow, state *ml.SlidingWindow) []durationConversion { + return []durationConversion{ + shadowedDuration("delay", input.Delay, &state.Delay), + shadowedDuration("offset", input.Offset, &state.Offset), + inPlaceDuration("window_duration", &state.WindowDuration), + inPlaceDuration("slide_duration", &state.SlideDuration), + } +} + +func tumblingWindowDurations(input *resources.FeatureTumblingWindow, state *ml.TumblingWindow) []durationConversion { + return []durationConversion{ + shadowedDuration("delay", input.Delay, &state.Delay), + shadowedDuration("offset", input.Offset, &state.Offset), + inPlaceDuration("window_duration", &state.WindowDuration), + } +} + +func continuousWindowDurations(state *ml.ContinuousWindow) []durationConversion { + return []durationConversion{ + inPlaceDuration("offset", &state.Offset), + inPlaceDuration("window_duration", &state.WindowDuration), + } +} + // featureState converts the bundle config for a feature into the API type, parsing every duration. // The config declares durations as strings; see the override note in // bundle/config/resources/feature.go. @@ -247,12 +339,8 @@ func rollingWindowState(input *resources.FeatureRollingWindow) (*ml.RollingWindo } state := input.RollingWindow - var err error - if state.Delay, err = optionalDuration(input.Delay); err != nil { - return nil, fmt.Errorf("delay: %w", err) - } - if state.WindowDuration, err = optionalDuration(input.WindowDuration); err != nil { - return nil, fmt.Errorf("window_duration: %w", err) + if err := convertDurations(rollingWindowDurations(input, &state)); err != nil { + return nil, err } return &state, nil @@ -264,12 +352,8 @@ func sawtoothWindowState(input *resources.FeatureSawtoothWindow) (*ml.SawtoothWi } state := input.SawtoothWindow - var err error - if state.Delay, err = optionalDuration(input.Delay); err != nil { - return nil, fmt.Errorf("delay: %w", err) - } - if state.WindowDuration, err = optionalDuration(input.WindowDuration); err != nil { - return nil, fmt.Errorf("window_duration: %w", err) + if err := convertDurations(sawtoothWindowDurations(input, &state)); err != nil { + return nil, err } return &state, nil @@ -281,18 +365,8 @@ func slidingWindowState(input *resources.FeatureSlidingWindow) (*ml.SlidingWindo } state := input.SlidingWindow - var err error - if state.Delay, err = optionalDuration(input.Delay); err != nil { - return nil, fmt.Errorf("delay: %w", err) - } - if state.Offset, err = optionalDuration(input.Offset); err != nil { - return nil, fmt.Errorf("offset: %w", err) - } - if state.WindowDuration, err = optionalCanonicalDuration(state.WindowDuration); err != nil { - return nil, fmt.Errorf("window_duration: %w", err) - } - if state.SlideDuration, err = optionalCanonicalDuration(state.SlideDuration); err != nil { - return nil, fmt.Errorf("slide_duration: %w", err) + if err := convertDurations(slidingWindowDurations(input, &state)); err != nil { + return nil, err } return &state, nil @@ -304,15 +378,8 @@ func tumblingWindowState(input *resources.FeatureTumblingWindow) (*ml.TumblingWi } state := input.TumblingWindow - var err error - if state.Delay, err = optionalDuration(input.Delay); err != nil { - return nil, fmt.Errorf("delay: %w", err) - } - if state.Offset, err = optionalDuration(input.Offset); err != nil { - return nil, fmt.Errorf("offset: %w", err) - } - if state.WindowDuration, err = optionalCanonicalDuration(state.WindowDuration); err != nil { - return nil, fmt.Errorf("window_duration: %w", err) + if err := convertDurations(tumblingWindowDurations(input, &state)); err != nil { + return nil, err } return &state, nil @@ -324,12 +391,8 @@ func continuousWindowState(input *ml.ContinuousWindow) (*ml.ContinuousWindow, er } state := *input - var err error - if state.Offset, err = optionalCanonicalDuration(state.Offset); err != nil { - return nil, fmt.Errorf("offset: %w", err) - } - if state.WindowDuration, err = optionalCanonicalDuration(state.WindowDuration); err != nil { - return nil, fmt.Errorf("window_duration: %w", err) + if err := convertDurations(continuousWindowDurations(&state)); err != nil { + return nil, err } return &state, nil @@ -353,9 +416,8 @@ func featureSourceLatenessState(input *resources.FeatureSourceLateness) (*ml.Sou } state := input.SourceLateness - var err error - if state.SettlingDelay, err = optionalDuration(input.SettlingDelay); err != nil { - return nil, fmt.Errorf("settling_delay: %w", err) + if err := convertDurations(featureSourceLatenessDurations(input, &state)); err != nil { + return nil, err } return &state, nil diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go index 564916c0a13..8f0461f1ce1 100644 --- a/bundle/direct/dresources/feature_duration_test.go +++ b/bundle/direct/dresources/feature_duration_test.go @@ -253,6 +253,61 @@ func isDurationTyped(typ reflect.Type) bool { return typ == durationType } +// featureDurationRegistries returns every registry keyed by the json path its fields hang from in +// ml.Feature. Each window is reached by two paths, the deprecated top-level time_window and the one +// under the aggregation function, so each window registry is listed under both. +func featureDurationRegistries() map[string][]durationConversion { + windows := map[string][]durationConversion{ + "rolling": rollingWindowDurations(&resources.FeatureRollingWindow{}, &ml.RollingWindow{}), + "sawtooth": sawtoothWindowDurations(&resources.FeatureSawtoothWindow{}, &ml.SawtoothWindow{}), + "sliding": slidingWindowDurations(&resources.FeatureSlidingWindow{}, &ml.SlidingWindow{}), + "tumbling": tumblingWindowDurations(&resources.FeatureTumblingWindow{}, &ml.TumblingWindow{}), + "continuous": continuousWindowDurations(&ml.ContinuousWindow{}), + } + + registries := map[string][]durationConversion{ + "source.lateness": featureSourceLatenessDurations(&resources.FeatureSourceLateness{}, &ml.SourceLateness{}), + } + for _, timeWindow := range []string{"time_window", "function.aggregation_function.time_window"} { + for window, conversions := range windows { + registries[timeWindow+"."+window] = conversions + } + } + return registries +} + +// TestFeatureDurationRegistryIsComplete fails when the API gains a duration field, or a window +// carrying one, that no registry converts, and when a registry still names a field the API dropped. +// A field whose SDK type flips between *sdkduration.Duration and string is reported by the compiler +// instead, because shadowedDuration and inPlaceDuration take different pointer types. +func TestFeatureDurationRegistryIsComplete(t *testing.T) { + registered := map[string]bool{} + for path, conversions := range featureDurationRegistries() { + for _, conversion := range conversions { + registered[path+"."+conversion.field] = true + } + } + + found := 0 + for path, typ := range jsonLeaves(reflect.TypeFor[ml.Feature]()) { + if !isDurationTyped(typ) && !(typ.Kind() == reflect.String && durationNamed(path)) { + continue + } + found++ + assert.Truef(t, registered[path], + "ml.Feature has duration field %q that no registry converts; add one line to the registry "+ + "in bundle/direct/dresources/feature_duration.go", path) + delete(registered, path) + } + assert.Positive(t, found, "walked no duration fields, the walker is broken") + + var stale []string + for path := range registered { + stale = append(stale, path) + } + assert.Emptyf(t, stale, "the registry converts fields ml.Feature does not have: %v", stale) +} + // TestFeatureDurationFieldsAreRetyped fails when the API gains a duration field the bundle config // does not re-type as a string, including one on a window kind that does not exist yet. func TestFeatureDurationFieldsAreRetyped(t *testing.T) { From 61a6ca1797dac0830af1621db59b73d355497d68 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 01:17:08 +0000 Subject: [PATCH 40/47] [FS-2621] Convert a window through one generic function instead of five The five per-window conversion functions differed only in their two types and which registry they walked, so adding a window meant writing another copy. featureTimeWindowState now calls one generic function per window, passing the registry and an accessor for the embedded API type. The accessor exists because Go does not permit field access on a type parameter, so the one line that reads input. cannot be written generically. A continuous window has no config wrapper, since both its durations are strings the config does not shadow, so its registry takes the same pair as the others and ignores the first. Co-authored-by: Isaac --- bundle/direct/dresources/feature_duration.go | 105 ++++++------------ .../dresources/feature_duration_test.go | 2 +- 2 files changed, 35 insertions(+), 72 deletions(-) diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index 0a56c21deba..74757c9d639 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -205,6 +205,23 @@ func convertDurations(conversions []durationConversion) error { return nil } +// windowState copies the API type a window's config wrapper embeds and converts its durations. +// embedded returns that copy because Go does not permit field access on a type parameter. +func windowState[Config, State any]( + input *Config, + embedded func(*Config) State, + durations func(*Config, *State) []durationConversion, +) (*State, error) { + if input == nil { + return nil, nil + } + state := embedded(input) + if err := convertDurations(durations(input, &state)); err != nil { + return nil, err + } + return &state, nil +} + // The registry: every duration field the adapter converts, one entry per field. A spec change that // adds a duration field is one line here. @@ -245,7 +262,8 @@ func tumblingWindowDurations(input *resources.FeatureTumblingWindow, state *ml.T } } -func continuousWindowDurations(state *ml.ContinuousWindow) []durationConversion { +// The config does not shadow a continuous window's durations, so there is nothing to read from it. +func continuousWindowDurations(_, state *ml.ContinuousWindow) []durationConversion { return []durationConversion{ inPlaceDuration("offset", &state.Offset), inPlaceDuration("window_duration", &state.WindowDuration), @@ -314,90 +332,35 @@ func featureTimeWindowState(input *resources.FeatureTimeWindow) (*ml.TimeWindow, state := input.TimeWindow var err error - if state.Rolling, err = rollingWindowState(input.Rolling); err != nil { + if state.Rolling, err = windowState(input.Rolling, + func(c *resources.FeatureRollingWindow) ml.RollingWindow { return c.RollingWindow }, + rollingWindowDurations); err != nil { return nil, fmt.Errorf("rolling: %w", err) } - if state.Sawtooth, err = sawtoothWindowState(input.Sawtooth); err != nil { + if state.Sawtooth, err = windowState(input.Sawtooth, + func(c *resources.FeatureSawtoothWindow) ml.SawtoothWindow { return c.SawtoothWindow }, + sawtoothWindowDurations); err != nil { return nil, fmt.Errorf("sawtooth: %w", err) } - if state.Sliding, err = slidingWindowState(input.Sliding); err != nil { + if state.Sliding, err = windowState(input.Sliding, + func(c *resources.FeatureSlidingWindow) ml.SlidingWindow { return c.SlidingWindow }, + slidingWindowDurations); err != nil { return nil, fmt.Errorf("sliding: %w", err) } - if state.Tumbling, err = tumblingWindowState(input.Tumbling); err != nil { + if state.Tumbling, err = windowState(input.Tumbling, + func(c *resources.FeatureTumblingWindow) ml.TumblingWindow { return c.TumblingWindow }, + tumblingWindowDurations); err != nil { return nil, fmt.Errorf("tumbling: %w", err) } - if state.Continuous, err = continuousWindowState(state.Continuous); err != nil { + if state.Continuous, err = windowState(state.Continuous, + func(c *ml.ContinuousWindow) ml.ContinuousWindow { return *c }, + continuousWindowDurations); err != nil { return nil, fmt.Errorf("continuous: %w", err) } return &state, nil } -func rollingWindowState(input *resources.FeatureRollingWindow) (*ml.RollingWindow, error) { - if input == nil { - return nil, nil - } - state := input.RollingWindow - - if err := convertDurations(rollingWindowDurations(input, &state)); err != nil { - return nil, err - } - - return &state, nil -} - -func sawtoothWindowState(input *resources.FeatureSawtoothWindow) (*ml.SawtoothWindow, error) { - if input == nil { - return nil, nil - } - state := input.SawtoothWindow - - if err := convertDurations(sawtoothWindowDurations(input, &state)); err != nil { - return nil, err - } - - return &state, nil -} - -func slidingWindowState(input *resources.FeatureSlidingWindow) (*ml.SlidingWindow, error) { - if input == nil { - return nil, nil - } - state := input.SlidingWindow - - if err := convertDurations(slidingWindowDurations(input, &state)); err != nil { - return nil, err - } - - return &state, nil -} - -func tumblingWindowState(input *resources.FeatureTumblingWindow) (*ml.TumblingWindow, error) { - if input == nil { - return nil, nil - } - state := input.TumblingWindow - - if err := convertDurations(tumblingWindowDurations(input, &state)); err != nil { - return nil, err - } - - return &state, nil -} - -func continuousWindowState(input *ml.ContinuousWindow) (*ml.ContinuousWindow, error) { - if input == nil { - return nil, nil - } - state := *input - - if err := convertDurations(continuousWindowDurations(&state)); err != nil { - return nil, err - } - - return &state, nil -} - func featureDataSourceState(input *resources.FeatureDataSource) (*ml.DataSource, error) { state := input.DataSource diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go index 8f0461f1ce1..6d8e0a7ce24 100644 --- a/bundle/direct/dresources/feature_duration_test.go +++ b/bundle/direct/dresources/feature_duration_test.go @@ -262,7 +262,7 @@ func featureDurationRegistries() map[string][]durationConversion { "sawtooth": sawtoothWindowDurations(&resources.FeatureSawtoothWindow{}, &ml.SawtoothWindow{}), "sliding": slidingWindowDurations(&resources.FeatureSlidingWindow{}, &ml.SlidingWindow{}), "tumbling": tumblingWindowDurations(&resources.FeatureTumblingWindow{}, &ml.TumblingWindow{}), - "continuous": continuousWindowDurations(&ml.ContinuousWindow{}), + "continuous": continuousWindowDurations(nil, &ml.ContinuousWindow{}), } registries := map[string][]durationConversion{ From 789d7a9e4d03775529c2d390a35fa0588ce8bd28 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 19:15:51 +0000 Subject: [PATCH 41/47] [FS-2621] Name the duration helpers after what they convert Each helper was named for its mechanism rather than its subject, so a reader had to open it to learn which of the two duration field types it served. The pair of registry constructors now name the field type they take, the walker names what it walks, and the conversion functions read as " with converted durations". The generic window function also documents its three parameters, including why an accessor closure is needed at all. Co-authored-by: Isaac --- bundle/direct/dresources/feature_duration.go | 125 +++++++++--------- .../dresources/feature_duration_test.go | 14 +- 2 files changed, 71 insertions(+), 68 deletions(-) diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index 74757c9d639..adb09fc428f 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -130,8 +130,9 @@ func parseExtendedDuration(s string) (time.Duration, error) { return total, nil } -// optionalDuration converts a configured duration into the API type, leaving an unset value and an -// unresolved reference alone. +// optionalDuration converts a duration composed of one or more units (s, h, d, w) into the API type +// that represents duration strictly in seconds, leaving an unset value and an unresolved reference +// alone. func optionalDuration(s string) (*sdkduration.Duration, error) { if s == "" || dynvar.IsPureVariableReference(s) { return nil, nil @@ -139,8 +140,9 @@ func optionalDuration(s string) (*sdkduration.Duration, error) { return duration(s) } -// optionalCanonicalDuration renders a duration the API types as a string in the seconds form the -// API answers with, under the same two exemptions as optionalDuration. +// optionalCanonicalDuration converts a duration composed of one or more units (s, h, d, w) into an +// untyped string of the form "604800s", leaving an unset value and an unresolved reference alone. +// The only difference with optionalDuration is the return type. func optionalCanonicalDuration(s string) (string, error) { if s == "" || dynvar.IsPureVariableReference(s) { return s, nil @@ -167,8 +169,8 @@ type durationConversion struct { apply func() error } -// shadowedDuration parses the string the bundle config shadows an SDK duration field with. -func shadowedDuration(field, source string, dest **sdkduration.Duration) durationConversion { +// durationConversionForNonStringTypedDuration parses a non-string typed duration. +func durationConversionForNonStringTypedDuration(field, source string, dest **sdkduration.Duration) durationConversion { return durationConversion{field: field, apply: func() error { parsed, err := optionalDuration(source) if err != nil { @@ -179,8 +181,8 @@ func shadowedDuration(field, source string, dest **sdkduration.Duration) duratio }} } -// inPlaceDuration rewrites a duration the SDK types as a string, which the config does not shadow. -func inPlaceDuration(field string, dest *string) durationConversion { +// durationConversionForStringTypedDuration parses a string typed duration. +func durationConversionForStringTypedDuration(field string, dest *string) durationConversion { return durationConversion{field: field, apply: func() error { rewritten, err := optionalCanonicalDuration(*dest) if err != nil { @@ -191,12 +193,9 @@ func inPlaceDuration(field string, dest *string) durationConversion { }} } -// convertDurations applies a registry, naming the field that failed. -// -// A field whose SDK type flips between *sdkduration.Duration and string has to move between -// shadowedDuration and inPlaceDuration, which take different pointer types, so the compiler reports -// it. A duration field the API adds is reported by TestFeatureDurationRegistryIsComplete. -func convertDurations(conversions []durationConversion) error { +// applyDurationConversions walks the registry of duration conversions for each field, applies the +// duration conversion function and reports the field that failed. +func applyDurationConversions(conversions []durationConversion) error { for _, conversion := range conversions { if err := conversion.apply(); err != nil { return fmt.Errorf("%s: %w", conversion.field, err) @@ -205,18 +204,22 @@ func convertDurations(conversions []durationConversion) error { return nil } -// windowState copies the API type a window's config wrapper embeds and converts its durations. -// embedded returns that copy because Go does not permit field access on a type parameter. -func windowState[Config, State any]( +// windowWithConvertedDurations copies the API type a window's config wrapper embeds and converts +// its durations to be strictly in seconds. +// +// input is the config-wrapped window. embedded selects the embedded API type from it, which Go does +// not permit generically: a field on a type parameter is inaccessible. durationConversionsForWindow +// is this window's registry. +func windowWithConvertedDurations[Config, State any]( input *Config, embedded func(*Config) State, - durations func(*Config, *State) []durationConversion, + durationConversionsForWindow func(*Config, *State) []durationConversion, ) (*State, error) { if input == nil { return nil, nil } state := embedded(input) - if err := convertDurations(durations(input, &state)); err != nil { + if err := applyDurationConversions(durationConversionsForWindow(input, &state)); err != nil { return nil, err } return &state, nil @@ -225,48 +228,48 @@ func windowState[Config, State any]( // The registry: every duration field the adapter converts, one entry per field. A spec change that // adds a duration field is one line here. -func featureSourceLatenessDurations(input *resources.FeatureSourceLateness, state *ml.SourceLateness) []durationConversion { +func durationConversionsForSourceLateness(input *resources.FeatureSourceLateness, state *ml.SourceLateness) []durationConversion { return []durationConversion{ - shadowedDuration("settling_delay", input.SettlingDelay, &state.SettlingDelay), + durationConversionForNonStringTypedDuration("settling_delay", input.SettlingDelay, &state.SettlingDelay), } } -func rollingWindowDurations(input *resources.FeatureRollingWindow, state *ml.RollingWindow) []durationConversion { +func durationConversionsForRollingWindow(input *resources.FeatureRollingWindow, state *ml.RollingWindow) []durationConversion { return []durationConversion{ - shadowedDuration("delay", input.Delay, &state.Delay), - shadowedDuration("window_duration", input.WindowDuration, &state.WindowDuration), + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("window_duration", input.WindowDuration, &state.WindowDuration), } } -func sawtoothWindowDurations(input *resources.FeatureSawtoothWindow, state *ml.SawtoothWindow) []durationConversion { +func durationConversionsForSawtoothWindow(input *resources.FeatureSawtoothWindow, state *ml.SawtoothWindow) []durationConversion { return []durationConversion{ - shadowedDuration("delay", input.Delay, &state.Delay), - shadowedDuration("window_duration", input.WindowDuration, &state.WindowDuration), + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("window_duration", input.WindowDuration, &state.WindowDuration), } } -func slidingWindowDurations(input *resources.FeatureSlidingWindow, state *ml.SlidingWindow) []durationConversion { +func durationConversionsForSlidingWindow(input *resources.FeatureSlidingWindow, state *ml.SlidingWindow) []durationConversion { return []durationConversion{ - shadowedDuration("delay", input.Delay, &state.Delay), - shadowedDuration("offset", input.Offset, &state.Offset), - inPlaceDuration("window_duration", &state.WindowDuration), - inPlaceDuration("slide_duration", &state.SlideDuration), + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("offset", input.Offset, &state.Offset), + durationConversionForStringTypedDuration("window_duration", &state.WindowDuration), + durationConversionForStringTypedDuration("slide_duration", &state.SlideDuration), } } -func tumblingWindowDurations(input *resources.FeatureTumblingWindow, state *ml.TumblingWindow) []durationConversion { +func durationConversionsForTumblingWindow(input *resources.FeatureTumblingWindow, state *ml.TumblingWindow) []durationConversion { return []durationConversion{ - shadowedDuration("delay", input.Delay, &state.Delay), - shadowedDuration("offset", input.Offset, &state.Offset), - inPlaceDuration("window_duration", &state.WindowDuration), + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("offset", input.Offset, &state.Offset), + durationConversionForStringTypedDuration("window_duration", &state.WindowDuration), } } // The config does not shadow a continuous window's durations, so there is nothing to read from it. -func continuousWindowDurations(_, state *ml.ContinuousWindow) []durationConversion { +func durationConversionsForContinuousWindow(_, state *ml.ContinuousWindow) []durationConversion { return []durationConversion{ - inPlaceDuration("offset", &state.Offset), - inPlaceDuration("window_duration", &state.WindowDuration), + durationConversionForStringTypedDuration("offset", &state.Offset), + durationConversionForStringTypedDuration("window_duration", &state.WindowDuration), } } @@ -276,13 +279,13 @@ func continuousWindowDurations(_, state *ml.ContinuousWindow) []durationConversi func featureState(input *resources.Feature) (*ml.Feature, error) { state := input.Feature - function, err := featureFunctionState(&input.Function) + function, err := featureFunctionWithConvertedDurations(&input.Function) if err != nil { return nil, fmt.Errorf("function: %w", err) } state.Function = *function - source, err := featureDataSourceState(&input.Source) + source, err := featureDataSourceWithConvertedDurations(&input.Source) if err != nil { return nil, fmt.Errorf("source: %w", err) } @@ -290,7 +293,7 @@ func featureState(input *resources.Feature) (*ml.Feature, error) { // The deprecated top-level time_window is converted on the same terms as the one under the // aggregation function. - state.TimeWindow, err = featureTimeWindowState(input.TimeWindow) + state.TimeWindow, err = featureTimeWindowWithConvertedDurations(input.TimeWindow) if err != nil { return nil, fmt.Errorf("time_window: %w", err) } @@ -298,10 +301,10 @@ func featureState(input *resources.Feature) (*ml.Feature, error) { return &state, nil } -func featureFunctionState(input *resources.FeatureFunction) (*ml.Function, error) { +func featureFunctionWithConvertedDurations(input *resources.FeatureFunction) (*ml.Function, error) { state := input.Function - aggregation, err := featureAggregationFunctionState(input.AggregationFunction) + aggregation, err := featureAggregationFunctionWithConvertedDurations(input.AggregationFunction) if err != nil { return nil, fmt.Errorf("aggregation_function: %w", err) } @@ -310,13 +313,13 @@ func featureFunctionState(input *resources.FeatureFunction) (*ml.Function, error return &state, nil } -func featureAggregationFunctionState(input *resources.FeatureAggregationFunction) (*ml.AggregationFunction, error) { +func featureAggregationFunctionWithConvertedDurations(input *resources.FeatureAggregationFunction) (*ml.AggregationFunction, error) { if input == nil { return nil, nil } state := input.AggregationFunction - timeWindow, err := featureTimeWindowState(input.TimeWindow) + timeWindow, err := featureTimeWindowWithConvertedDurations(input.TimeWindow) if err != nil { return nil, fmt.Errorf("time_window: %w", err) } @@ -325,46 +328,46 @@ func featureAggregationFunctionState(input *resources.FeatureAggregationFunction return &state, nil } -func featureTimeWindowState(input *resources.FeatureTimeWindow) (*ml.TimeWindow, error) { +func featureTimeWindowWithConvertedDurations(input *resources.FeatureTimeWindow) (*ml.TimeWindow, error) { if input == nil { return nil, nil } state := input.TimeWindow var err error - if state.Rolling, err = windowState(input.Rolling, + if state.Rolling, err = windowWithConvertedDurations(input.Rolling, func(c *resources.FeatureRollingWindow) ml.RollingWindow { return c.RollingWindow }, - rollingWindowDurations); err != nil { + durationConversionsForRollingWindow); err != nil { return nil, fmt.Errorf("rolling: %w", err) } - if state.Sawtooth, err = windowState(input.Sawtooth, + if state.Sawtooth, err = windowWithConvertedDurations(input.Sawtooth, func(c *resources.FeatureSawtoothWindow) ml.SawtoothWindow { return c.SawtoothWindow }, - sawtoothWindowDurations); err != nil { + durationConversionsForSawtoothWindow); err != nil { return nil, fmt.Errorf("sawtooth: %w", err) } - if state.Sliding, err = windowState(input.Sliding, + if state.Sliding, err = windowWithConvertedDurations(input.Sliding, func(c *resources.FeatureSlidingWindow) ml.SlidingWindow { return c.SlidingWindow }, - slidingWindowDurations); err != nil { + durationConversionsForSlidingWindow); err != nil { return nil, fmt.Errorf("sliding: %w", err) } - if state.Tumbling, err = windowState(input.Tumbling, + if state.Tumbling, err = windowWithConvertedDurations(input.Tumbling, func(c *resources.FeatureTumblingWindow) ml.TumblingWindow { return c.TumblingWindow }, - tumblingWindowDurations); err != nil { + durationConversionsForTumblingWindow); err != nil { return nil, fmt.Errorf("tumbling: %w", err) } - if state.Continuous, err = windowState(state.Continuous, + if state.Continuous, err = windowWithConvertedDurations(state.Continuous, func(c *ml.ContinuousWindow) ml.ContinuousWindow { return *c }, - continuousWindowDurations); err != nil { + durationConversionsForContinuousWindow); err != nil { return nil, fmt.Errorf("continuous: %w", err) } return &state, nil } -func featureDataSourceState(input *resources.FeatureDataSource) (*ml.DataSource, error) { +func featureDataSourceWithConvertedDurations(input *resources.FeatureDataSource) (*ml.DataSource, error) { state := input.DataSource - lateness, err := featureSourceLatenessState(input.Lateness) + lateness, err := featureSourceLatenessWithConvertedDurations(input.Lateness) if err != nil { return nil, fmt.Errorf("lateness: %w", err) } @@ -373,13 +376,13 @@ func featureDataSourceState(input *resources.FeatureDataSource) (*ml.DataSource, return &state, nil } -func featureSourceLatenessState(input *resources.FeatureSourceLateness) (*ml.SourceLateness, error) { +func featureSourceLatenessWithConvertedDurations(input *resources.FeatureSourceLateness) (*ml.SourceLateness, error) { if input == nil { return nil, nil } state := input.SourceLateness - if err := convertDurations(featureSourceLatenessDurations(input, &state)); err != nil { + if err := applyDurationConversions(durationConversionsForSourceLateness(input, &state)); err != nil { return nil, err } diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go index 6d8e0a7ce24..d5e986314c4 100644 --- a/bundle/direct/dresources/feature_duration_test.go +++ b/bundle/direct/dresources/feature_duration_test.go @@ -258,15 +258,15 @@ func isDurationTyped(typ reflect.Type) bool { // under the aggregation function, so each window registry is listed under both. func featureDurationRegistries() map[string][]durationConversion { windows := map[string][]durationConversion{ - "rolling": rollingWindowDurations(&resources.FeatureRollingWindow{}, &ml.RollingWindow{}), - "sawtooth": sawtoothWindowDurations(&resources.FeatureSawtoothWindow{}, &ml.SawtoothWindow{}), - "sliding": slidingWindowDurations(&resources.FeatureSlidingWindow{}, &ml.SlidingWindow{}), - "tumbling": tumblingWindowDurations(&resources.FeatureTumblingWindow{}, &ml.TumblingWindow{}), - "continuous": continuousWindowDurations(nil, &ml.ContinuousWindow{}), + "rolling": durationConversionsForRollingWindow(&resources.FeatureRollingWindow{}, &ml.RollingWindow{}), + "sawtooth": durationConversionsForSawtoothWindow(&resources.FeatureSawtoothWindow{}, &ml.SawtoothWindow{}), + "sliding": durationConversionsForSlidingWindow(&resources.FeatureSlidingWindow{}, &ml.SlidingWindow{}), + "tumbling": durationConversionsForTumblingWindow(&resources.FeatureTumblingWindow{}, &ml.TumblingWindow{}), + "continuous": durationConversionsForContinuousWindow(nil, &ml.ContinuousWindow{}), } registries := map[string][]durationConversion{ - "source.lateness": featureSourceLatenessDurations(&resources.FeatureSourceLateness{}, &ml.SourceLateness{}), + "source.lateness": durationConversionsForSourceLateness(&resources.FeatureSourceLateness{}, &ml.SourceLateness{}), } for _, timeWindow := range []string{"time_window", "function.aggregation_function.time_window"} { for window, conversions := range windows { @@ -279,7 +279,7 @@ func featureDurationRegistries() map[string][]durationConversion { // TestFeatureDurationRegistryIsComplete fails when the API gains a duration field, or a window // carrying one, that no registry converts, and when a registry still names a field the API dropped. // A field whose SDK type flips between *sdkduration.Duration and string is reported by the compiler -// instead, because shadowedDuration and inPlaceDuration take different pointer types. +// instead, because the two registry constructors take different pointer types. func TestFeatureDurationRegistryIsComplete(t *testing.T) { registered := map[string]bool{} for path, conversions := range featureDurationRegistries() { From 989ea04f9199b91280cac392ffa98254c9042a7b Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 19:42:21 +0000 Subject: [PATCH 42/47] [FS-2621] Convert source lateness through the same generic function Source lateness has the same shape as a window -- a config wrapper embedding the API type plus a registry -- so its own conversion function was a sixth copy of one that already existed. It now calls the generic one, which is no longer window-specific and is named for the field it converts. Co-authored-by: Isaac --- bundle/direct/dresources/feature_duration.go | 43 ++++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index adb09fc428f..289a3f813cf 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -204,22 +204,22 @@ func applyDurationConversions(conversions []durationConversion) error { return nil } -// windowWithConvertedDurations copies the API type a window's config wrapper embeds and converts -// its durations to be strictly in seconds. +// fieldWithConvertedDurations copies the API type a config wrapper embeds and converts its +// durations to be strictly in seconds. // -// input is the config-wrapped window. embedded selects the embedded API type from it, which Go does -// not permit generically: a field on a type parameter is inaccessible. durationConversionsForWindow -// is this window's registry. -func windowWithConvertedDurations[Config, State any]( +// input is the config-wrapped field. embedded selects the embedded API type from it, which Go does +// not permit generically: a field on a type parameter is inaccessible. durationConversionsForField +// is that field's registry. +func fieldWithConvertedDurations[Config, State any]( input *Config, embedded func(*Config) State, - durationConversionsForWindow func(*Config, *State) []durationConversion, + durationConversionsForField func(*Config, *State) []durationConversion, ) (*State, error) { if input == nil { return nil, nil } state := embedded(input) - if err := applyDurationConversions(durationConversionsForWindow(input, &state)); err != nil { + if err := applyDurationConversions(durationConversionsForField(input, &state)); err != nil { return nil, err } return &state, nil @@ -335,27 +335,27 @@ func featureTimeWindowWithConvertedDurations(input *resources.FeatureTimeWindow) state := input.TimeWindow var err error - if state.Rolling, err = windowWithConvertedDurations(input.Rolling, + if state.Rolling, err = fieldWithConvertedDurations(input.Rolling, func(c *resources.FeatureRollingWindow) ml.RollingWindow { return c.RollingWindow }, durationConversionsForRollingWindow); err != nil { return nil, fmt.Errorf("rolling: %w", err) } - if state.Sawtooth, err = windowWithConvertedDurations(input.Sawtooth, + if state.Sawtooth, err = fieldWithConvertedDurations(input.Sawtooth, func(c *resources.FeatureSawtoothWindow) ml.SawtoothWindow { return c.SawtoothWindow }, durationConversionsForSawtoothWindow); err != nil { return nil, fmt.Errorf("sawtooth: %w", err) } - if state.Sliding, err = windowWithConvertedDurations(input.Sliding, + if state.Sliding, err = fieldWithConvertedDurations(input.Sliding, func(c *resources.FeatureSlidingWindow) ml.SlidingWindow { return c.SlidingWindow }, durationConversionsForSlidingWindow); err != nil { return nil, fmt.Errorf("sliding: %w", err) } - if state.Tumbling, err = windowWithConvertedDurations(input.Tumbling, + if state.Tumbling, err = fieldWithConvertedDurations(input.Tumbling, func(c *resources.FeatureTumblingWindow) ml.TumblingWindow { return c.TumblingWindow }, durationConversionsForTumblingWindow); err != nil { return nil, fmt.Errorf("tumbling: %w", err) } - if state.Continuous, err = windowWithConvertedDurations(state.Continuous, + if state.Continuous, err = fieldWithConvertedDurations(state.Continuous, func(c *ml.ContinuousWindow) ml.ContinuousWindow { return *c }, durationConversionsForContinuousWindow); err != nil { return nil, fmt.Errorf("continuous: %w", err) @@ -367,7 +367,9 @@ func featureTimeWindowWithConvertedDurations(input *resources.FeatureTimeWindow) func featureDataSourceWithConvertedDurations(input *resources.FeatureDataSource) (*ml.DataSource, error) { state := input.DataSource - lateness, err := featureSourceLatenessWithConvertedDurations(input.Lateness) + lateness, err := fieldWithConvertedDurations(input.Lateness, + func(c *resources.FeatureSourceLateness) ml.SourceLateness { return c.SourceLateness }, + durationConversionsForSourceLateness) if err != nil { return nil, fmt.Errorf("lateness: %w", err) } @@ -375,16 +377,3 @@ func featureDataSourceWithConvertedDurations(input *resources.FeatureDataSource) return &state, nil } - -func featureSourceLatenessWithConvertedDurations(input *resources.FeatureSourceLateness) (*ml.SourceLateness, error) { - if input == nil { - return nil, nil - } - state := input.SourceLateness - - if err := applyDurationConversions(durationConversionsForSourceLateness(input, &state)); err != nil { - return nil, err - } - - return &state, nil -} From 241322c58fe49681696a30b4a21ff6b962215ff3 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 20:17:30 +0000 Subject: [PATCH 43/47] [FS-2621] Replace the reflective duration guard with a written checklist The guard walked ml.Feature and resources.Feature with a hand-rolled reimplementation of encoding/json's field-hiding rules to decide whether every duration field was re-typed, shadowed and converted. It cost 280 lines of reflection that nobody reading the adapter could follow. The registry now states, in order, what adding a duration field requires. What is given up is detection of a field the API adds: TestFeatureStateConvertsEveryWindow still fails when an existing field stops being converted, but a new field reaches nothing until someone follows step 4. Co-authored-by: Isaac --- bundle/direct/dresources/feature_duration.go | 18 +- .../dresources/feature_duration_test.go | 289 +----------------- 2 files changed, 18 insertions(+), 289 deletions(-) diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index 289a3f813cf..76e91b22842 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -225,8 +225,22 @@ func fieldWithConvertedDurations[Config, State any]( return &state, nil } -// The registry: every duration field the adapter converts, one entry per field. A spec change that -// adds a duration field is one line here. +// The registry: every duration field the adapter converts, one entry per field. +// +// When the API gains a duration field, nothing detects it. Add it here by hand: +// +// 1. If the SDK types the field as *sdkduration.Duration, re-type it as a string in +// bundle/config/resources/feature.go so a bundle can write "7d". A field the SDK already types +// as a string needs no config change. +// 2. Add one entry to the durationConversionsFor below that owns the field, picking +// durationConversionForNonStringTypedDuration or durationConversionForStringTypedDuration to +// match the SDK's type for it. +// 3. If the field sits on a message that has no registry yet, write one, then convert that message +// through fieldWithConvertedDurations from whichever conversion owns it: featureState, +// featureFunctionWithConvertedDurations, featureAggregationFunctionWithConvertedDurations, +// featureTimeWindowWithConvertedDurations or featureDataSourceWithConvertedDurations. +// 4. Set the field in featureWithEveryDuration and assert it in +// TestFeatureStateConvertsEveryWindow, which is the only check that the field is converted. func durationConversionsForSourceLateness(input *resources.FeatureSourceLateness, state *ml.SourceLateness) []durationConversion { return []durationConversion{ diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go index d5e986314c4..0670b31c403 100644 --- a/bundle/direct/dresources/feature_duration_test.go +++ b/bundle/direct/dresources/feature_duration_test.go @@ -1,15 +1,10 @@ package dresources import ( - "reflect" - "strings" "testing" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/structs/structtag" sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -151,288 +146,8 @@ func TestFeaturePrepareInputConfigPassesConfigThrough(t *testing.T) { assert.Nil(t, got.Refs) } -// ---- the guard ---- - -// durationNamed reports whether a string-typed field holds a duration. The API types five of these -// as plain strings, and neither the Go type nor the generated schema says so, so the name is the -// only signal. -func durationNamed(jsonName string) bool { - leaf := jsonName - if i := strings.LastIndex(leaf, "."); i >= 0 { - leaf = leaf[i+1:] - } - // freshness_target is the one duration-shaped name that follows no convention, so it is named. - return strings.HasSuffix(leaf, "_duration") || strings.HasSuffix(leaf, "delay") || - leaf == "offset" || leaf == "freshness_target" -} - -var durationType = reflect.TypeFor[sdkduration.Duration]() - -func terminalStruct(typ reflect.Type) bool { - return typ == durationType || typ == reflect.TypeFor[sdktime.Time]() || - typ == reflect.TypeFor[sdkfieldmask.FieldMask]() -} - -// flattenedFields returns typ's fields the way encoding/json sees them: breadth-first from the -// outer struct inward, with a JSON name declared at a shallower depth hiding a deeper one, matching -// buildStructInfo in libs/dyn/convert/struct_info.go. -func flattenedFields(typ reflect.Type) []reflect.StructField { - var out []reflect.StructField - seen := map[string]bool{} - queue := []reflect.Type{typ} - - for len(queue) > 0 { - cur := queue[0] - queue = queue[1:] - - var embedded []reflect.Type - for i := range cur.NumField() { - field := cur.Field(i) - name := structtag.JSONTag(field.Tag.Get("json")).Name() - - if field.Anonymous && name == "" { - inner := field.Type - for inner.Kind() == reflect.Pointer { - inner = inner.Elem() - } - if inner.Kind() == reflect.Struct { - embedded = append(embedded, inner) - continue - } - } - if !field.IsExported() || name == "" || name == "-" || seen[name] { - continue - } - seen[name] = true - out = append(out, field) - } - queue = append(queue, embedded...) - } - return out -} - -// jsonLeaves maps the JSON path of every leaf reachable from typ to the leaf's Go type. -func jsonLeaves(typ reflect.Type) map[string]reflect.Type { - out := map[string]reflect.Type{} - collectJSONLeaves(typ, "", map[reflect.Type]bool{}, out) - return out -} - -func collectJSONLeaves(typ reflect.Type, prefix string, onPath map[reflect.Type]bool, out map[string]reflect.Type) { - for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice { - typ = typ.Elem() - } - if typ.Kind() != reflect.Struct || onPath[typ] { - return - } - onPath[typ] = true - defer delete(onPath, typ) - - for _, field := range flattenedFields(typ) { - path := structtag.JSONTag(field.Tag.Get("json")).Name() - if prefix != "" { - path = prefix + "." + path - } - - inner := field.Type - for inner.Kind() == reflect.Pointer || inner.Kind() == reflect.Slice { - inner = inner.Elem() - } - if inner.Kind() == reflect.Struct && !terminalStruct(inner) { - collectJSONLeaves(inner, path, onPath, out) - continue - } - out[path] = field.Type - } -} - -func isDurationTyped(typ reflect.Type) bool { - for typ.Kind() == reflect.Pointer { - typ = typ.Elem() - } - return typ == durationType -} - -// featureDurationRegistries returns every registry keyed by the json path its fields hang from in -// ml.Feature. Each window is reached by two paths, the deprecated top-level time_window and the one -// under the aggregation function, so each window registry is listed under both. -func featureDurationRegistries() map[string][]durationConversion { - windows := map[string][]durationConversion{ - "rolling": durationConversionsForRollingWindow(&resources.FeatureRollingWindow{}, &ml.RollingWindow{}), - "sawtooth": durationConversionsForSawtoothWindow(&resources.FeatureSawtoothWindow{}, &ml.SawtoothWindow{}), - "sliding": durationConversionsForSlidingWindow(&resources.FeatureSlidingWindow{}, &ml.SlidingWindow{}), - "tumbling": durationConversionsForTumblingWindow(&resources.FeatureTumblingWindow{}, &ml.TumblingWindow{}), - "continuous": durationConversionsForContinuousWindow(nil, &ml.ContinuousWindow{}), - } - - registries := map[string][]durationConversion{ - "source.lateness": durationConversionsForSourceLateness(&resources.FeatureSourceLateness{}, &ml.SourceLateness{}), - } - for _, timeWindow := range []string{"time_window", "function.aggregation_function.time_window"} { - for window, conversions := range windows { - registries[timeWindow+"."+window] = conversions - } - } - return registries -} - -// TestFeatureDurationRegistryIsComplete fails when the API gains a duration field, or a window -// carrying one, that no registry converts, and when a registry still names a field the API dropped. -// A field whose SDK type flips between *sdkduration.Duration and string is reported by the compiler -// instead, because the two registry constructors take different pointer types. -func TestFeatureDurationRegistryIsComplete(t *testing.T) { - registered := map[string]bool{} - for path, conversions := range featureDurationRegistries() { - for _, conversion := range conversions { - registered[path+"."+conversion.field] = true - } - } - - found := 0 - for path, typ := range jsonLeaves(reflect.TypeFor[ml.Feature]()) { - if !isDurationTyped(typ) && !(typ.Kind() == reflect.String && durationNamed(path)) { - continue - } - found++ - assert.Truef(t, registered[path], - "ml.Feature has duration field %q that no registry converts; add one line to the registry "+ - "in bundle/direct/dresources/feature_duration.go", path) - delete(registered, path) - } - assert.Positive(t, found, "walked no duration fields, the walker is broken") - - var stale []string - for path := range registered { - stale = append(stale, path) - } - assert.Emptyf(t, stale, "the registry converts fields ml.Feature does not have: %v", stale) -} - -// TestFeatureDurationFieldsAreRetyped fails when the API gains a duration field the bundle config -// does not re-type as a string, including one on a window kind that does not exist yet. -func TestFeatureDurationFieldsAreRetyped(t *testing.T) { - sdk := jsonLeaves(reflect.TypeFor[ml.Feature]()) - config := jsonLeaves(reflect.TypeFor[resources.Feature]()) - - found := 0 - for path, typ := range sdk { - if !isDurationTyped(typ) { - continue - } - found++ - configType, ok := config[path] - require.Truef(t, ok, "ml.Feature has duration field %q, missing from resources.Feature", path) - assert.Equalf(t, reflect.String, configType.Kind(), - "%q must be re-typed as a string in resources.Feature so a bundle can write 7d; "+ - "add a wrapper in bundle/config/resources/feature.go and convert it in featureState", - path) - } - assert.Positive(t, found, "walked no duration fields, the walker is broken") -} - -// TestFeatureShadowsAreNotStale fails when a re-typed field no longer matches a field of the API -// type, which is what an upstream rename leaves behind. -func TestFeatureShadowsAreNotStale(t *testing.T) { - sdkFields := map[string]bool{} - for _, field := range flattenedFields(reflect.TypeFor[ml.Feature]()) { - sdkFields[structtag.JSONTag(field.Tag.Get("json")).Name()] = true - } - - // The subtrees resources.Feature re-types are those whose top-level name it redeclares. - shadowed := map[string]bool{} - configType := reflect.TypeFor[resources.Feature]() - for i := range configType.NumField() { - field := configType.Field(i) - name := structtag.JSONTag(field.Tag.Get("json")).Name() - if !field.Anonymous && sdkFields[name] { - shadowed[name] = true - } - } - require.NotEmpty(t, shadowed, "found no re-typed subtree, the walker is broken") - - sdk := jsonLeaves(reflect.TypeFor[ml.Feature]()) - for path := range jsonLeaves(configType) { - if !shadowed[strings.SplitN(path, ".", 2)[0]] { - continue - } - _, ok := sdk[path] - assert.Truef(t, ok, "resources.Feature declares %q but ml.Feature does not; a re-typed field is stale", path) - } -} - -// TestFeatureStateConvertsEveryDurationField is the coverage guard: every duration field the API -// declares must come out of featureState holding the configured value. A field that is re-typed but -// never converted arrives nil and fails here. -func TestFeatureStateConvertsEveryDurationField(t *testing.T) { - state, err := featureState(featureWithEveryDuration()) - require.NoError(t, err) - - checked := 0 - for path, typ := range jsonLeaves(reflect.TypeFor[ml.Feature]()) { - if !isDurationTyped(typ) && !(typ.Kind() == reflect.String && durationNamed(path)) { - continue - } - checked++ - value := leafByPath(t, reflect.ValueOf(state), path) - require.Truef(t, value.IsValid(), "%q is unreachable in the converted state", path) - - if isDurationTyped(typ) { - require.Falsef(t, value.IsNil(), - "%q came out unset: add it to featureState and to featureWithEveryDuration", path) - assert.Equalf(t, 86400.0, value.Interface().(*sdkduration.Duration).AsDuration().Seconds(), path) - continue - } - assert.Equalf(t, "86400s", value.String(), - "%q came out unconverted: add it to featureState and to featureWithEveryDuration", path) - } - assert.Positive(t, checked, "walked no duration fields, the walker is broken") -} - -// TestMaterializationSetDurationFieldsAreKnown fails when the API gives a materialization set a -// second duration-shaped field. freshness_target has no reader in the service, so the resource -// deliberately converts nothing; a new field forces that decision to be revisited. -func TestMaterializationSetDurationFieldsAreKnown(t *testing.T) { - var found []string - for path, typ := range jsonLeaves(reflect.TypeFor[ml.MaterializationSet]()) { - if isDurationTyped(typ) || (typ.Kind() == reflect.String && durationNamed(path)) { - found = append(found, path) - } - } - assert.ElementsMatch(t, []string{"materialized_features.streaming_mode.freshness_target"}, found) -} - -// leafByPath resolves a dotted JSON path against a struct value, following the same shadowing rule -// as flattenedFields. -func leafByPath(t *testing.T, v reflect.Value, path string) reflect.Value { - t.Helper() - for _, segment := range strings.Split(path, ".") { - for v.Kind() == reflect.Pointer { - if v.IsNil() { - return reflect.Value{} - } - v = v.Elem() - } - if v.Kind() != reflect.Struct { - return reflect.Value{} - } - var next reflect.Value - for _, field := range flattenedFields(v.Type()) { - if structtag.JSONTag(field.Tag.Get("json")).Name() != segment { - continue - } - next = v.FieldByIndex(field.Index) - break - } - if !next.IsValid() { - return reflect.Value{} - } - v = next - } - return v -} - -// featureWithEveryDuration sets every duration field a feature has to one day, so the coverage -// guard can assert each one survives conversion. +// featureWithEveryDuration sets every duration field a feature has to one day, so +// TestFeatureStateConvertsEveryWindow can assert each one survives conversion. func featureWithEveryDuration() *resources.Feature { const day = "1d" From b4e589c21272f7780869192a97409383dd6827ec Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 20:37:38 +0000 Subject: [PATCH 44/47] [FS-2621] Cover both shorthand fields and three duration forms The shorthand case exercised one authoring field alone and one alongside members written out in full. It now also carries the batch pair shorthand as a whole set on its own, and both fields in one set. That last arm found a gap: the service refuses a set whose two expansions ask for different cron schedule modes, which a sawtooth entry and a batch pair always do, and the fake did not know the rule -- so it answered with the batch-versus- streaming refusal that comes later. validateSingleCronScheduleMode mirrors the real rule, before the operation is reserved, since the service refuses with 400 rather than by settling the record. The fixtures now write the feature windows three ways, so the duration parser is exercised through a live create in each: seconds alone, extended units alone, and units mixed. Verified against a staging workspace through the daniel-bai-fs2680 swap. Co-authored-by: Isaac --- .../resources/materialization_sets/README.md | 2 +- .../shorthand/batch-only.yml.tmpl | 41 +++++++++++ .../shorthand/batch-pair.yml.tmpl | 8 +-- .../shorthand/both-shorthand.yml.tmpl | 72 +++++++++++++++++++ .../shorthand/databricks.yml.tmpl | 2 +- .../shorthand/manual-schedule.yml.tmpl | 2 +- .../shorthand/moved.yml.tmpl | 2 +- .../materialization_sets/shorthand/output.txt | 65 ++++++++++++++++- .../materialization_sets/shorthand/script | 15 ++++ libs/testserver/materialization_set.go | 24 +++++++ 10 files changed, 223 insertions(+), 10 deletions(-) create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl create mode 100644 acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md index 6a2c0ae39fd..f6156c0e553 100644 --- a/acceptance/bundle/resources/materialization_sets/README.md +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -68,7 +68,7 @@ zero with its state dropped. The deployment-history leg is the one path that rec | `batch` | a batch feature with both destinations creates and settles; neither member of its pair can be removed alone, and removing both empties the set, which the update endpoint refuses in favour of delete; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not; a set mixing a batch-path feature with a streaming-path one is refused at create | | `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing; the three members are one cascade group, so each of the three strict subsets of them is refused -- with the message depending on whether the removal names the member a delete is applied to -- and removing all three empties the set, which the update endpoint refuses | | `settled_failure` | a failure the server hits after reserving an operation comes back as HTTP 200 with the operation FAILED: the adapter recovers the error code from the operation and the per-member outcome from its metadata, the update fails naming each member left behind and writes nothing, and the destroy reports success anyway because the engine forgives a delete error it cannot classify | -| `shorthand` | a feature's whole materialization declared once in `sawtooth_materialized_features` expands into its three members, and in `batch_offline_and_online_materialized_features` into its two; neither authoring field survives into the summary, a redeploy plans nothing, and one edit to the online destination reaches both online members while the offline one stays put. A batch schedule the service refuses is reported by the service naming the feature, not by the CLI -- the expansion judges nothing it is given. The last arm mixes a shorthand entry with members written out in full, so the expansion is appended rather than substituted | +| `shorthand` | a feature's whole materialization declared once in `sawtooth_materialized_features` expands into its three members, and in `batch_offline_and_online_materialized_features` into its two; neither authoring field survives into the summary, a redeploy plans nothing, and one edit to the online destination reaches both online members while the offline one stays put. A batch schedule the service refuses is reported by the service naming the feature, not by the CLI -- the expansion judges nothing it is given. One arm mixes a shorthand entry with members written out in full, so the expansion is appended rather than substituted; another carries the batch pair shorthand alone, with no member written out. The last declares both authoring fields in one set, which the service refuses because a sawtooth entry has to carry a DERIVED schedule and a batch pair a MANUAL one, and one request cannot mix cron schedule modes. Its fixtures write the feature windows three ways: seconds only, extended units alone (`3d`), and units mixed (`2d1h35m40s`) | | `streaming` | a streaming online member needs no schedule and no cron; declaring a schedule state on it, or repointing its destination, plans as a `recreate` rather than an in-place update; with two features materialized to one destination and no offline member between them, each online member is its own cascade group, so one can be removed alone as an `update` | ## Running against a workspace diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl new file mode 100644 index 00000000000..0b60914a07f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + pairfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.pair_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # The shorthand is the whole declaration: every member of this set was produced by expanding + # the one entry below. + batch_offline_and_online_materialized_features: + - feature_name: ${resources.features.pairfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl index 1fb2d6fface..a0ce3b72f32 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl @@ -14,8 +14,8 @@ resources: input: value time_window: sliding: - window_duration: 604800s - slide_duration: 86400s + window_duration: 2d1h35m40s + slide_duration: 1h30m entities: - name: id timeseries_column: @@ -31,8 +31,8 @@ resources: input: value time_window: sliding: - window_duration: 604800s - slide_duration: 86400s + window_duration: 2d1h35m40s + slide_duration: 1h30m entities: - name: id timeseries_column: diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl new file mode 100644 index 00000000000..070f265c555 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl @@ -0,0 +1,72 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + bothsaw: + full_name: $CATALOG_NAME.$SCHEMA_NAME.bothsaw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 3d + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + bothbatch: + full_name: $CATALOG_NAME.$SCHEMA_NAME.bothbatch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Both shorthand fields at once. The two entries name identical destinations, so what the + # service refuses is the pair of cron schedule modes the two expansions ask for. + sawtooth_materialized_features: + - feature_name: ${resources.features.bothsaw.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + batch_offline_and_online_materialized_features: + - feature_name: ${resources.features.bothbatch.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl index 8e97eb27faf..3577aa6f0d1 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl @@ -14,7 +14,7 @@ resources: input: value.passenger_count_1 time_window: sawtooth: - window_duration: 259200s + window_duration: 3d entities: - name: value.pickup_zip timeseries_column: diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl index 438249cca84..e1947c6e714 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl @@ -14,7 +14,7 @@ resources: input: value.passenger_count_1 time_window: sawtooth: - window_duration: 259200s + window_duration: 3d entities: - name: value.pickup_zip timeseries_column: diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl index 0ff83c0eb16..dbd5ed03cf8 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl +++ b/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl @@ -14,7 +14,7 @@ resources: input: value.passenger_count_1 time_window: sawtooth: - window_duration: 259200s + window_duration: 3d entities: - name: value.pickup_zip timeseries_column: diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/output.txt b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt index e1fc72dff49..66ed29c2d3f 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/output.txt +++ b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt @@ -11,7 +11,7 @@ HTTP Status: 200 API error_code: BAD_REQUEST API message: materialization set [UUID] operation failed: Feature 'danielb.fs2680.shd_[UNIQUE_NAME]' must declare a DERIVED cron schedule on its batch materialize feature requests. The sawtooth batch materializations refresh on a daily schedule the service computes, so a caller neither pins nor omits it. -Files: 5 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted === Declaring the same feature once with a DERIVED batch schedule creates the three members it === is made of, and neither shorthand field survives into the summary @@ -98,7 +98,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorth Created features.otherfeat Created features.pairfeat Created materialization_sets.set1 -Files: 6 uploaded, 0 deleted +Files: 8 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> members @@ -128,3 +128,64 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ] } + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.otherfeat + delete resources.features.pairfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default + +Destroy: 3 deleted + +=== The batch pair shorthand carries a whole set on its own, with no member written out +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.pairfeat +Created materialization_sets.set1 +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> members +{ + "sawtooth_shorthand_remains": false, + "batch_pair_shorthand_remains": false, + "members": [ + { + "feature": "pairfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "pairfeat", + "destination": "online", + "trigger": "batch" + } + ] +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.pairfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default + +Destroy: 2 deleted + +=== Both shorthand fields expand in the same set, which the service then refuses: a sawtooth +=== entry has to carry a DERIVED schedule and a batch pair a MANUAL one, and one request cannot +=== mix the two modes. The CLI expanded both fields and judged neither. +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.bothbatch +Created features.bothsaw +Error: cannot create resources.materialization_sets.set1: All cron-scheduled features in a single request must use the same cron schedule mode; a request cannot mix DERIVED (service-derived) and MANUAL (caller-supplied) schedules. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: All cron-scheduled features in a single request must use the same cron schedule mode; a request cannot mix DERIVED (service-derived) and MANUAL (caller-supplied) schedules. + +Files: 8 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/script b/acceptance/bundle/resources/materialization_sets/shorthand/script index 41c0e390756..b0f0f4284d4 100644 --- a/acceptance/bundle/resources/materialization_sets/shorthand/script +++ b/acceptance/bundle/resources/materialization_sets/shorthand/script @@ -76,4 +76,19 @@ render batch-pair.yml.tmpl trace $CLI bundle deploy trace members +trace $CLI bundle destroy --auto-approve + +title "The batch pair shorthand carries a whole set on its own, with no member written out" +render batch-only.yml.tmpl +trace $CLI bundle deploy +trace members + +trace $CLI bundle destroy --auto-approve + +title "Both shorthand fields expand in the same set, which the service then refuses: a sawtooth" +title "entry has to carry a DERIVED schedule and a batch pair a MANUAL one, and one request cannot" +title "mix the two modes. The CLI expanded both fields and judged neither." +render both-shorthand.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + rm -f out.plan.json diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go index 04266193087..3eb19af1988 100644 --- a/libs/testserver/materialization_set.go +++ b/libs/testserver/materialization_set.go @@ -112,6 +112,27 @@ func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Resp return nil } +// validateSingleCronScheduleMode mirrors the create-path rule that one request carries one cron +// schedule mode. A sawtooth feature must be DERIVED and a batch-only feature MANUAL, so this is +// what a set holding both runs into first. +func validateSingleCronScheduleMode(members []ml.MaterializedFeature) *Response { + modes := map[ml.CronScheduleMode]bool{} + for _, member := range members { + if member.CronScheduleTrigger == nil { + continue + } + modes[member.CronScheduleTrigger.Mode] = true + } + if len(modes) > 1 { + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "All cron-scheduled features in a single request must use the same cron schedule "+ + "mode; a request cannot mix DERIVED (service-derived) and MANUAL "+ + "(caller-supplied) schedules.") + return &response + } + return nil +} + // featureMaterializationDestination renders the destination pair the create compares across // features: a feature's offline config plus the online config it is also materialized to, if any. func featureMaterializationDestination(members []ml.MaterializedFeature) string { @@ -323,6 +344,9 @@ func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { return *bad } + if bad := validateSingleCronScheduleMode(set.MaterializedFeatures); bad != nil { + return *bad + } set.MaterializationSetId = nextUUID() // The operation record is reserved here, so what follows is the operation's outcome rather From 74f5f4f4571146e7cbf15904271285497f8f0147 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 20:59:22 +0000 Subject: [PATCH 45/47] [FS-2621] Record that the expansion mutates typed state The mutator writes Go structs, and the Python entry point merges the dynamic tree back once the normalize sequence returns. That tree only picks up a typed mutation when the next mutator enters, so the expansion reaches the merge today because other entries follow it in the sequence -- a dependency on ordering that nothing else records. Co-authored-by: Isaac --- .../expand_materialization_set_materialized_feature_members.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go index d6201542eb6..bc54c59b672 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go @@ -97,6 +97,9 @@ func batchOfflineAndOnlineMaterializedFeatureMembers(entry ml.MaterializedFeatur } } +// Apply mutates typed state, and the Python path merges the dynamic tree back once the normalize +// sequence returns, so this must not be the last entry in applyNormalizeMutators: that tree picks up +// a typed mutation only when the next mutator enters. func (m *expandMaterializationSetMaterializedFeatureMembers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for _, set := range b.Config.Resources.MaterializationSets { if set == nil { From a6ad3b6850669e398cffa10bd439c57d27960ade Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 21:27:47 +0000 Subject: [PATCH 46/47] [FS-2621] Name the feature config wrappers after the API types they re-type pydabs-codegen names a Python class after the schema definition it comes from, and a definition is named after its Go type. Prefixing these wrappers therefore renamed every Python class for the features resource -- AggregationFunction to FeatureAggregationFunction and eight more -- which shipped in release v1.22.0. Dropping the prefix keeps the published names. Verified by regenerating: the schema definitions are resources.AggregationFunction and friends, and the features package exports exactly what v1.22.0 exports. Co-authored-by: Isaac --- bundle/config/resources/feature.go | 74 +++++++++---------- bundle/direct/dresources/feature_duration.go | 28 +++---- .../dresources/feature_duration_test.go | 40 +++++----- 3 files changed, 71 insertions(+), 71 deletions(-) diff --git a/bundle/config/resources/feature.go b/bundle/config/resources/feature.go index be0140e14b1..715bcddbd9b 100644 --- a/bundle/config/resources/feature.go +++ b/bundle/config/resources/feature.go @@ -22,138 +22,138 @@ type Feature struct { // === overrides over [ml.Feature] ========= // ========================================= - Function FeatureFunction `json:"function"` - Source FeatureDataSource `json:"source"` - TimeWindow *FeatureTimeWindow `json:"time_window,omitempty"` + Function Function `json:"function"` + Source DataSource `json:"source"` + TimeWindow *TimeWindow `json:"time_window,omitempty"` } // The types below re-type the API's duration fields as strings so a bundle can write a duration // with day and hour units; featureState converts them before deploy. Each embeds the API type and // shadows only the fields it re-types, so every other field resolves through the embedding. -type FeatureFunction struct { +type Function struct { ml.Function - AggregationFunction *FeatureAggregationFunction `json:"aggregation_function,omitempty"` + AggregationFunction *AggregationFunction `json:"aggregation_function,omitempty"` } -type FeatureAggregationFunction struct { +type AggregationFunction struct { ml.AggregationFunction - TimeWindow *FeatureTimeWindow `json:"time_window,omitempty"` + TimeWindow *TimeWindow `json:"time_window,omitempty"` } -type FeatureTimeWindow struct { +type TimeWindow struct { ml.TimeWindow - Rolling *FeatureRollingWindow `json:"rolling,omitempty"` - Sawtooth *FeatureSawtoothWindow `json:"sawtooth,omitempty"` - Sliding *FeatureSlidingWindow `json:"sliding,omitempty"` - Tumbling *FeatureTumblingWindow `json:"tumbling,omitempty"` + Rolling *RollingWindow `json:"rolling,omitempty"` + Sawtooth *SawtoothWindow `json:"sawtooth,omitempty"` + Sliding *SlidingWindow `json:"sliding,omitempty"` + Tumbling *TumblingWindow `json:"tumbling,omitempty"` } -type FeatureRollingWindow struct { +type RollingWindow struct { ml.RollingWindow Delay string `json:"delay,omitempty"` WindowDuration string `json:"window_duration,omitempty"` } -type FeatureSawtoothWindow struct { +type SawtoothWindow struct { ml.SawtoothWindow Delay string `json:"delay,omitempty"` WindowDuration string `json:"window_duration,omitempty"` } -type FeatureSlidingWindow struct { +type SlidingWindow struct { ml.SlidingWindow Delay string `json:"delay,omitempty"` Offset string `json:"offset,omitempty"` } -type FeatureTumblingWindow struct { +type TumblingWindow struct { ml.TumblingWindow Delay string `json:"delay,omitempty"` Offset string `json:"offset,omitempty"` } -type FeatureDataSource struct { +type DataSource struct { ml.DataSource - Lateness *FeatureSourceLateness `json:"lateness,omitempty"` + Lateness *SourceLateness `json:"lateness,omitempty"` } -type FeatureSourceLateness struct { +type SourceLateness struct { ml.SourceLateness SettlingDelay string `json:"settling_delay,omitempty"` } // Each wrapper declares its own JSON methods, overriding the ones its embedded API type promotes. -func (c *FeatureFunction) UnmarshalJSON(b []byte) error { +func (c *Function) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureFunction) MarshalJSON() ([]byte, error) { +func (c Function) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureAggregationFunction) UnmarshalJSON(b []byte) error { +func (c *AggregationFunction) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureAggregationFunction) MarshalJSON() ([]byte, error) { +func (c AggregationFunction) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureTimeWindow) UnmarshalJSON(b []byte) error { +func (c *TimeWindow) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureTimeWindow) MarshalJSON() ([]byte, error) { +func (c TimeWindow) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureRollingWindow) UnmarshalJSON(b []byte) error { +func (c *RollingWindow) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureRollingWindow) MarshalJSON() ([]byte, error) { +func (c RollingWindow) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureSawtoothWindow) UnmarshalJSON(b []byte) error { +func (c *SawtoothWindow) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureSawtoothWindow) MarshalJSON() ([]byte, error) { +func (c SawtoothWindow) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureSlidingWindow) UnmarshalJSON(b []byte) error { +func (c *SlidingWindow) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureSlidingWindow) MarshalJSON() ([]byte, error) { +func (c SlidingWindow) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureTumblingWindow) UnmarshalJSON(b []byte) error { +func (c *TumblingWindow) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureTumblingWindow) MarshalJSON() ([]byte, error) { +func (c TumblingWindow) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureDataSource) UnmarshalJSON(b []byte) error { +func (c *DataSource) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureDataSource) MarshalJSON() ([]byte, error) { +func (c DataSource) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } -func (c *FeatureSourceLateness) UnmarshalJSON(b []byte) error { +func (c *SourceLateness) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, c) } -func (c FeatureSourceLateness) MarshalJSON() ([]byte, error) { +func (c SourceLateness) MarshalJSON() ([]byte, error) { return marshal.Marshal(c) } diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go index 76e91b22842..4f9a0d5d46e 100644 --- a/bundle/direct/dresources/feature_duration.go +++ b/bundle/direct/dresources/feature_duration.go @@ -242,27 +242,27 @@ func fieldWithConvertedDurations[Config, State any]( // 4. Set the field in featureWithEveryDuration and assert it in // TestFeatureStateConvertsEveryWindow, which is the only check that the field is converted. -func durationConversionsForSourceLateness(input *resources.FeatureSourceLateness, state *ml.SourceLateness) []durationConversion { +func durationConversionsForSourceLateness(input *resources.SourceLateness, state *ml.SourceLateness) []durationConversion { return []durationConversion{ durationConversionForNonStringTypedDuration("settling_delay", input.SettlingDelay, &state.SettlingDelay), } } -func durationConversionsForRollingWindow(input *resources.FeatureRollingWindow, state *ml.RollingWindow) []durationConversion { +func durationConversionsForRollingWindow(input *resources.RollingWindow, state *ml.RollingWindow) []durationConversion { return []durationConversion{ durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), durationConversionForNonStringTypedDuration("window_duration", input.WindowDuration, &state.WindowDuration), } } -func durationConversionsForSawtoothWindow(input *resources.FeatureSawtoothWindow, state *ml.SawtoothWindow) []durationConversion { +func durationConversionsForSawtoothWindow(input *resources.SawtoothWindow, state *ml.SawtoothWindow) []durationConversion { return []durationConversion{ durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), durationConversionForNonStringTypedDuration("window_duration", input.WindowDuration, &state.WindowDuration), } } -func durationConversionsForSlidingWindow(input *resources.FeatureSlidingWindow, state *ml.SlidingWindow) []durationConversion { +func durationConversionsForSlidingWindow(input *resources.SlidingWindow, state *ml.SlidingWindow) []durationConversion { return []durationConversion{ durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), durationConversionForNonStringTypedDuration("offset", input.Offset, &state.Offset), @@ -271,7 +271,7 @@ func durationConversionsForSlidingWindow(input *resources.FeatureSlidingWindow, } } -func durationConversionsForTumblingWindow(input *resources.FeatureTumblingWindow, state *ml.TumblingWindow) []durationConversion { +func durationConversionsForTumblingWindow(input *resources.TumblingWindow, state *ml.TumblingWindow) []durationConversion { return []durationConversion{ durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), durationConversionForNonStringTypedDuration("offset", input.Offset, &state.Offset), @@ -315,7 +315,7 @@ func featureState(input *resources.Feature) (*ml.Feature, error) { return &state, nil } -func featureFunctionWithConvertedDurations(input *resources.FeatureFunction) (*ml.Function, error) { +func featureFunctionWithConvertedDurations(input *resources.Function) (*ml.Function, error) { state := input.Function aggregation, err := featureAggregationFunctionWithConvertedDurations(input.AggregationFunction) @@ -327,7 +327,7 @@ func featureFunctionWithConvertedDurations(input *resources.FeatureFunction) (*m return &state, nil } -func featureAggregationFunctionWithConvertedDurations(input *resources.FeatureAggregationFunction) (*ml.AggregationFunction, error) { +func featureAggregationFunctionWithConvertedDurations(input *resources.AggregationFunction) (*ml.AggregationFunction, error) { if input == nil { return nil, nil } @@ -342,7 +342,7 @@ func featureAggregationFunctionWithConvertedDurations(input *resources.FeatureAg return &state, nil } -func featureTimeWindowWithConvertedDurations(input *resources.FeatureTimeWindow) (*ml.TimeWindow, error) { +func featureTimeWindowWithConvertedDurations(input *resources.TimeWindow) (*ml.TimeWindow, error) { if input == nil { return nil, nil } @@ -350,22 +350,22 @@ func featureTimeWindowWithConvertedDurations(input *resources.FeatureTimeWindow) var err error if state.Rolling, err = fieldWithConvertedDurations(input.Rolling, - func(c *resources.FeatureRollingWindow) ml.RollingWindow { return c.RollingWindow }, + func(c *resources.RollingWindow) ml.RollingWindow { return c.RollingWindow }, durationConversionsForRollingWindow); err != nil { return nil, fmt.Errorf("rolling: %w", err) } if state.Sawtooth, err = fieldWithConvertedDurations(input.Sawtooth, - func(c *resources.FeatureSawtoothWindow) ml.SawtoothWindow { return c.SawtoothWindow }, + func(c *resources.SawtoothWindow) ml.SawtoothWindow { return c.SawtoothWindow }, durationConversionsForSawtoothWindow); err != nil { return nil, fmt.Errorf("sawtooth: %w", err) } if state.Sliding, err = fieldWithConvertedDurations(input.Sliding, - func(c *resources.FeatureSlidingWindow) ml.SlidingWindow { return c.SlidingWindow }, + func(c *resources.SlidingWindow) ml.SlidingWindow { return c.SlidingWindow }, durationConversionsForSlidingWindow); err != nil { return nil, fmt.Errorf("sliding: %w", err) } if state.Tumbling, err = fieldWithConvertedDurations(input.Tumbling, - func(c *resources.FeatureTumblingWindow) ml.TumblingWindow { return c.TumblingWindow }, + func(c *resources.TumblingWindow) ml.TumblingWindow { return c.TumblingWindow }, durationConversionsForTumblingWindow); err != nil { return nil, fmt.Errorf("tumbling: %w", err) } @@ -378,11 +378,11 @@ func featureTimeWindowWithConvertedDurations(input *resources.FeatureTimeWindow) return &state, nil } -func featureDataSourceWithConvertedDurations(input *resources.FeatureDataSource) (*ml.DataSource, error) { +func featureDataSourceWithConvertedDurations(input *resources.DataSource) (*ml.DataSource, error) { state := input.DataSource lateness, err := fieldWithConvertedDurations(input.Lateness, - func(c *resources.FeatureSourceLateness) ml.SourceLateness { return c.SourceLateness }, + func(c *resources.SourceLateness) ml.SourceLateness { return c.SourceLateness }, durationConversionsForSourceLateness) if err != nil { return nil, fmt.Errorf("lateness: %w", err) diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go index 0670b31c403..cdea98442ff 100644 --- a/bundle/direct/dresources/feature_duration_test.go +++ b/bundle/direct/dresources/feature_duration_test.go @@ -84,10 +84,10 @@ func TestFeatureStateConvertsEveryWindow(t *testing.T) { func TestFeatureStateAcceptsCompoundAndProtobufForms(t *testing.T) { input := &resources.Feature{} - input.Function.AggregationFunction = &resources.FeatureAggregationFunction{ - TimeWindow: &resources.FeatureTimeWindow{ - Sawtooth: &resources.FeatureSawtoothWindow{WindowDuration: "3d15m"}, - Sliding: &resources.FeatureSlidingWindow{}, + input.Function.AggregationFunction = &resources.AggregationFunction{ + TimeWindow: &resources.TimeWindow{ + Sawtooth: &resources.SawtoothWindow{WindowDuration: "3d15m"}, + Sliding: &resources.SlidingWindow{}, }, } input.Function.AggregationFunction.TimeWindow.Sliding.SlideDuration = "604800s" @@ -101,10 +101,10 @@ func TestFeatureStateAcceptsCompoundAndProtobufForms(t *testing.T) { func TestFeatureStateLeavesUnsetAndVariableReferences(t *testing.T) { input := &resources.Feature{} - input.Function.AggregationFunction = &resources.FeatureAggregationFunction{ - TimeWindow: &resources.FeatureTimeWindow{ - Rolling: &resources.FeatureRollingWindow{WindowDuration: "${var.window}"}, - Sliding: &resources.FeatureSlidingWindow{}, + input.Function.AggregationFunction = &resources.AggregationFunction{ + TimeWindow: &resources.TimeWindow{ + Rolling: &resources.RollingWindow{WindowDuration: "${var.window}"}, + Sliding: &resources.SlidingWindow{}, }, } @@ -126,9 +126,9 @@ func TestFeatureStateWithoutWindows(t *testing.T) { func TestFeaturePrepareInputConfigReportsBadDuration(t *testing.T) { input := &resources.Feature{} - input.Function.AggregationFunction = &resources.FeatureAggregationFunction{ - TimeWindow: &resources.FeatureTimeWindow{ - Rolling: &resources.FeatureRollingWindow{WindowDuration: "7 fortnights"}, + input.Function.AggregationFunction = &resources.AggregationFunction{ + TimeWindow: &resources.TimeWindow{ + Rolling: &resources.RollingWindow{WindowDuration: "7 fortnights"}, }, } @@ -151,12 +151,12 @@ func TestFeaturePrepareInputConfigPassesConfigThrough(t *testing.T) { func featureWithEveryDuration() *resources.Feature { const day = "1d" - window := func() *resources.FeatureTimeWindow { - w := &resources.FeatureTimeWindow{ - Rolling: &resources.FeatureRollingWindow{Delay: day, WindowDuration: day}, - Sawtooth: &resources.FeatureSawtoothWindow{Delay: day, WindowDuration: day}, - Sliding: &resources.FeatureSlidingWindow{Delay: day, Offset: day}, - Tumbling: &resources.FeatureTumblingWindow{Delay: day, Offset: day}, + window := func() *resources.TimeWindow { + w := &resources.TimeWindow{ + Rolling: &resources.RollingWindow{Delay: day, WindowDuration: day}, + Sawtooth: &resources.SawtoothWindow{Delay: day, WindowDuration: day}, + Sliding: &resources.SlidingWindow{Delay: day, Offset: day}, + Tumbling: &resources.TumblingWindow{Delay: day, Offset: day}, } w.Sliding.WindowDuration = day w.Sliding.SlideDuration = day @@ -166,10 +166,10 @@ func featureWithEveryDuration() *resources.Feature { } input := &resources.Feature{ - Function: resources.FeatureFunction{ - AggregationFunction: &resources.FeatureAggregationFunction{TimeWindow: window()}, + Function: resources.Function{ + AggregationFunction: &resources.AggregationFunction{TimeWindow: window()}, }, - Source: resources.FeatureDataSource{Lateness: &resources.FeatureSourceLateness{SettlingDelay: day}}, + Source: resources.DataSource{Lateness: &resources.SourceLateness{SettlingDelay: day}}, TimeWindow: window(), } return input From 0f467a9753f18adefd5ae6dcf30d120de3e00f11 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Sat, 24 Oct 2026 22:54:58 +0000 Subject: [PATCH 47/47] [FS-2621] Give each shorthand its own authoring type Both fields took ml.MaterializedFeature, which a shorthand entry cannot be: a materialized feature holds one destination and one trigger, both proto oneofs, while an entry holds both destinations and two trigger kinds and the expansion splits them. In YAML the mismatch is invisible because no type name is written; in PyDABs the author constructs MaterializedFeature(...) for a value no materialized feature may hold. SawtoothMaterializedFeatures and BatchOfflineAndOnlineMaterializedFeatures carry exactly the fields their shape permits. Neither declares materialized_feature_id, which the server assigns per member, and the batch pair declares no streaming mode, so writing one on a feature that materializes on the batch path is no longer expressible rather than refused by the service. The YAML keys are unchanged, so no fixture or golden moves. The coverage test now also catches a member field the API adds that no authoring type declares. Verified against a staging workspace through the daniel-bai-fs2680 swap. Co-authored-by: Isaac --- ...zation_set_materialized_feature_members.go | 7 +- ...erialized_feature_members_coverage_test.go | 85 ++++++++++++------- ...n_set_materialized_feature_members_test.go | 51 ++++++----- .../config/resources/materialization_set.go | 53 +++++++++++- 4 files changed, 131 insertions(+), 65 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go index bc54c59b672..82a94d54b41 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go @@ -4,6 +4,7 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -25,7 +26,7 @@ func (m *expandMaterializationSetMaterializedFeatureMembers) Name() string { // the same online destination. // // materialized_feature_id is absent here because it is assigned server side. -func sawtoothMaterializedFeatureMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { +func sawtoothMaterializedFeatureMembers(entry resources.SawtoothMaterializedFeatures) []ml.MaterializedFeature { return []ml.MaterializedFeature{ // Batch offline. { @@ -66,7 +67,7 @@ func sawtoothMaterializedFeatureMembers(entry ml.MaterializedFeature) []ml.Mater // batchOfflineAndOnlineMaterializedFeatureMembers builds two materializations of the specified // feature to the online and offline destination. -func batchOfflineAndOnlineMaterializedFeatureMembers(entry ml.MaterializedFeature) []ml.MaterializedFeature { +func batchOfflineAndOnlineMaterializedFeatureMembers(entry resources.BatchOfflineAndOnlineMaterializedFeatures) []ml.MaterializedFeature { return []ml.MaterializedFeature{ // Batch offline. { @@ -79,7 +80,6 @@ func batchOfflineAndOnlineMaterializedFeatureMembers(entry ml.MaterializedFeatur CronSchedule: entry.CronSchedule, CronScheduleTrigger: entry.CronScheduleTrigger, TableTrigger: entry.TableTrigger, - StreamingMode: entry.StreamingMode, }, // Batch online. { @@ -92,7 +92,6 @@ func batchOfflineAndOnlineMaterializedFeatureMembers(entry ml.MaterializedFeatur CronSchedule: entry.CronSchedule, CronScheduleTrigger: entry.CronScheduleTrigger, TableTrigger: entry.TableTrigger, - StreamingMode: entry.StreamingMode, }, } } diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go index bd4b5ce6275..17adc6eadd7 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go @@ -1,6 +1,7 @@ package resourcemutator_test import ( + "maps" "reflect" "testing" @@ -10,13 +11,11 @@ import ( "github.com/stretchr/testify/require" ) -// memberFieldsNotCarried are the member fields a shorthand entry never contributes to the members -// it expands into. All but the first are OUTPUT_ONLY in the spec, so the schema generator already -// keeps them out of the bundle schema and no config can set them. -// -// materialized_feature_id is the one field a shorthand entry can set and have ignored: the server -// assigns one per member, so a single declared id cannot belong to all of them. -var memberFieldsNotCarried = map[string]bool{ +// serverOwnedMemberFields are the member fields no authoring type declares. All but the first are +// OUTPUT_ONLY in the spec, so the schema generator keeps them out of the bundle schema; +// materialized_feature_id is left out because the server assigns one per member and a single +// declared id cannot belong to all of them. +var serverOwnedMemberFields = map[string]bool{ "MaterializedFeatureId": true, "MaterializationSetId": true, "IsOnline": true, @@ -35,14 +34,14 @@ var memberFieldsDerivedByConfigLayer = map[string]bool{ "ForceSendFields": true, } -// memberWithEveryFieldSet returns a materialized feature whose every field holds a non-zero value, -// so that a field missing from every expanded member is visible as a zero. -func memberWithEveryFieldSet(t *testing.T) ml.MaterializedFeature { +// entryWithEveryFieldSet returns an authoring entry whose every field holds a non-zero value, so +// that a field reaching no expanded member is visible as a zero. +func entryWithEveryFieldSet[T any](t *testing.T) T { t.Helper() - var member ml.MaterializedFeature - value := reflect.ValueOf(&member).Elem() - for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + var entry T + value := reflect.ValueOf(&entry).Elem() + for field := range reflect.TypeFor[T]().Fields() { target := value.FieldByName(field.Name) switch field.Type.Kind() { case reflect.String: @@ -54,9 +53,9 @@ func memberWithEveryFieldSet(t *testing.T) ml.MaterializedFeature { case reflect.Pointer: target.Set(reflect.New(field.Type.Elem())) case reflect.Map: - entry := reflect.MakeMap(field.Type) - entry.SetMapIndex(reflect.ValueOf("k"), reflect.ValueOf("v")) - target.Set(entry) + pair := reflect.MakeMap(field.Type) + pair.SetMapIndex(reflect.ValueOf("k"), reflect.ValueOf("v")) + target.Set(pair) case reflect.Slice: target.Set(reflect.MakeSlice(field.Type, 1, 1)) default: @@ -65,25 +64,46 @@ func memberWithEveryFieldSet(t *testing.T) ml.MaterializedFeature { field.Name, field.Type.Kind()) } } - return member + return entry } -// TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField keeps each expansion's field list -// exhaustive. Members are built by naming their fields rather than by copying the entry, so a -// member field added to the proto later reaches no expanded member and nothing else notices. -func TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField(t *testing.T) { - entry := memberWithEveryFieldSet(t) +func notCarried(extra ...string) map[string]bool { + out := maps.Clone(serverOwnedMemberFields) + for _, name := range extra { + out[name] = true + } + return out +} - for name, set := range map[string]*resources.MaterializationSet{ +// TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField keeps each +// expansion's field list exhaustive and each authoring type complete against the API. Members are +// built by naming their fields, and the authoring types are hand-written, so a member field added +// to the proto later reaches no expanded member and nothing else notices. +func TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField(t *testing.T) { + for name, tc := range map[string]struct { + set *resources.MaterializationSet + notCarried map[string]bool + }{ "sawtooth_materialized_features": { - SawtoothMaterializedFeatures: []ml.MaterializedFeature{entry}, + set: &resources.MaterializationSet{ + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{ + entryWithEveryFieldSet[resources.SawtoothMaterializedFeatures](t), + }, + }, + notCarried: notCarried(), }, "batch_offline_and_online_materialized_features": { - BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{entry}, + set: &resources.MaterializationSet{ + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{ + entryWithEveryFieldSet[resources.BatchOfflineAndOnlineMaterializedFeatures](t), + }, + }, + // A feature on the batch path has no streaming mode, so the type declares none. + notCarried: notCarried("StreamingMode"), }, } { t.Run(name, func(t *testing.T) { - expanded := expandSet(t, set) + expanded := expandSet(t, tc.set) require.NotEmpty(t, expanded.MaterializedFeatures) for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { @@ -99,16 +119,17 @@ func TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberFiel } } - if memberFieldsNotCarried[field.Name] { + if tc.notCarried[field.Name] { assert.Falsef(t, carried, - "field %s is listed in memberFieldsNotCarried but reached an expanded "+ - "member; drop it from that list or stop carrying it", field.Name) + "field %s is listed as not carried but reached an expanded member; drop it "+ + "from that list or stop carrying it", field.Name) continue } assert.Truef(t, carried, - "field %s reached no member of the %s expansion. Add it to the member "+ - "literals in expand_materialization_set_materialized_feature_members.go, or to "+ - "memberFieldsNotCarried with the reason it is server-owned", + "field %s reached no member of the %s expansion. Declare it on the authoring "+ + "type in bundle/config/resources/materialization_set.go and carry it in "+ + "expand_materialization_set_materialized_feature_members.go, or list it as "+ + "not carried with the reason it is server-owned", field.Name, name) } }) diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go index f9e61d170c8..6e2666b153a 100644 --- a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go @@ -53,7 +53,7 @@ func expandSet(t *testing.T, set *resources.MaterializationSet) *resources.Mater func TestExpandMaterializationSetMaterializedFeatureMembersSawtooth(t *testing.T) { set := expandSet(t, &resources.MaterializationSet{ MaterializationSet: ml.MaterializationSet{}, - SawtoothMaterializedFeatures: []ml.MaterializedFeature{{ + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{{ FeatureName: shorthandFeatureName, OfflineStoreConfig: shorthandOfflineStore(), OnlineStoreConfig: shorthandOnlineStore(), @@ -91,7 +91,7 @@ func TestExpandMaterializationSetMaterializedFeatureMembersSawtooth(t *testing.T func TestExpandMaterializationSetMaterializedFeatureMembersBatchOfflineAndOnline(t *testing.T) { trigger := &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr} set := expandSet(t, &resources.MaterializationSet{ - BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{{ FeatureName: shorthandFeatureName, OfflineStoreConfig: shorthandOfflineStore(), OnlineStoreConfig: shorthandOnlineStore(), @@ -132,8 +132,24 @@ func TestExpandMaterializationSetMaterializedFeatureMembersCarriesSharedFields(t } for name, set := range map[string]*resources.MaterializationSet{ - "sawtooth": {SawtoothMaterializedFeatures: []ml.MaterializedFeature{entry}}, - "batch pair": {BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{entry}}, + "sawtooth": {SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{{ + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + OnlineStoreConfig: entry.OnlineStoreConfig, + }}}, + "batch pair": {BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{{ + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + OnlineStoreConfig: entry.OnlineStoreConfig, + }}}, } { t.Run(name, func(t *testing.T) { expanded := expandSet(t, set) @@ -150,25 +166,6 @@ func TestExpandMaterializationSetMaterializedFeatureMembersCarriesSharedFields(t } } -// TestExpandMaterializationSetMaterializedFeatureMembersKeepsStreamingModeOnBatchPair pins that the batch pair carries -// a trigger it has no use for rather than discarding it, so the service reports the mistake instead -// of the shorthand swallowing it. -func TestExpandMaterializationSetMaterializedFeatureMembersKeepsStreamingModeOnBatchPair(t *testing.T) { - set := expandSet(t, &resources.MaterializationSet{ - BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ - FeatureName: shorthandFeatureName, - OfflineStoreConfig: shorthandOfflineStore(), - OnlineStoreConfig: shorthandOnlineStore(), - StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, - }}, - }) - - require.Len(t, set.MaterializedFeatures, 2) - for _, member := range set.MaterializedFeatures { - assert.Equal(t, ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm, member.StreamingMode.Mode) - } -} - func TestExpandMaterializationSetMaterializedFeatureMembersAppendsToDeclaredMembers(t *testing.T) { declared := ml.MaterializedFeature{ FeatureName: "main.myschema.other", @@ -179,14 +176,14 @@ func TestExpandMaterializationSetMaterializedFeatureMembersAppendsToDeclaredMemb MaterializationSet: ml.MaterializationSet{ MaterializedFeatures: []ml.MaterializedFeature{declared}, }, - SawtoothMaterializedFeatures: []ml.MaterializedFeature{{ + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{{ FeatureName: shorthandFeatureName, OfflineStoreConfig: shorthandOfflineStore(), OnlineStoreConfig: shorthandOnlineStore(), CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, }}, - BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{{ + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{{ FeatureName: "main.myschema.pair", OfflineStoreConfig: shorthandOfflineStore(), OnlineStoreConfig: shorthandOnlineStore(), @@ -217,8 +214,8 @@ func TestExpandMaterializationSetMaterializedFeatureMembersLeavesASetWithoutShor MaterializationSet: ml.MaterializationSet{ MaterializedFeatures: []ml.MaterializedFeature{declared}, }, - SawtoothMaterializedFeatures: []ml.MaterializedFeature{}, - BatchOfflineAndOnlineMaterializedFeatures: []ml.MaterializedFeature{}, + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{}, + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{}, }, } { t.Run(name, func(t *testing.T) { diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go index a9dec0bcd53..3d72cf45f18 100644 --- a/bundle/config/resources/materialization_set.go +++ b/bundle/config/resources/materialization_set.go @@ -22,12 +22,61 @@ type MaterializationSet struct { // sawtooth-window feature: an offline batch member, an online batch member and an online // streaming member. ExpandMaterializationSetMaterializedFeatureMembers fans each entry out // into three materialized features. - SawtoothMaterializedFeatures []ml.MaterializedFeature `json:"sawtooth_materialized_features,omitempty"` + SawtoothMaterializedFeatures []SawtoothMaterializedFeatures `json:"sawtooth_materialized_features,omitempty"` // BatchOfflineAndOnlineMaterializedFeatures declares, per entry, one feature materialized to // the online and offline destination. ExpandMaterializationSetMaterializedFeatureMembers fans // each entry out into two MaterializedFeatures. - BatchOfflineAndOnlineMaterializedFeatures []ml.MaterializedFeature `json:"batch_offline_and_online_materialized_features,omitempty"` + BatchOfflineAndOnlineMaterializedFeatures []BatchOfflineAndOnlineMaterializedFeatures `json:"batch_offline_and_online_materialized_features,omitempty"` +} + +// SawtoothMaterializedFeatures is the materialized features of one sawtooth-window feature, which +// the author declares once. ml.MaterializedFeature cannot stand in for it: a materialized feature +// holds one destination and one trigger, both declared as proto oneofs, while this holds both +// destinations and both trigger kinds and the expansion splits them across the three members. +// +// materialized_feature_id is absent because the server assigns one per member. +type SawtoothMaterializedFeatures struct { + FeatureName string `json:"feature_name"` + Tags map[string]string `json:"tags,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` + Notifications *ml.MaterializationNotifications `json:"notifications,omitempty"` + PipelineScheduleState ml.MaterializedFeaturePipelineScheduleState `json:"pipeline_schedule_state,omitempty"` + + OfflineStoreConfig *ml.OfflineStoreConfig `json:"offline_store_config,omitempty"` + OnlineStoreConfig *ml.OnlineStoreConfig `json:"online_store_config,omitempty"` + + // The batch members take the cron schedule and the table trigger; the streaming member takes + // the streaming mode. + CronSchedule string `json:"cron_schedule,omitempty"` + CronScheduleTrigger *ml.CronSchedule `json:"cron_schedule_trigger,omitempty"` + TableTrigger *ml.TableTrigger `json:"table_trigger,omitempty"` + StreamingMode *ml.StreamingMode `json:"streaming_mode,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +// BatchOfflineAndOnlineMaterializedFeatures is the materialized features of one feature taken to +// both stores on the batch path, which the author declares once. It carries no streaming mode: a +// feature on the batch path has none, and the two members it expands into differ only in their +// destination. +// +// materialized_feature_id is absent because the server assigns one per member. +type BatchOfflineAndOnlineMaterializedFeatures struct { + FeatureName string `json:"feature_name"` + Tags map[string]string `json:"tags,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` + Notifications *ml.MaterializationNotifications `json:"notifications,omitempty"` + PipelineScheduleState ml.MaterializedFeaturePipelineScheduleState `json:"pipeline_schedule_state,omitempty"` + + OfflineStoreConfig *ml.OfflineStoreConfig `json:"offline_store_config,omitempty"` + OnlineStoreConfig *ml.OnlineStoreConfig `json:"online_store_config,omitempty"` + + CronSchedule string `json:"cron_schedule,omitempty"` + CronScheduleTrigger *ml.CronSchedule `json:"cron_schedule_trigger,omitempty"` + TableTrigger *ml.TableTrigger `json:"table_trigger,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` } func (s *MaterializationSet) UnmarshalJSON(b []byte) error {