diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6f45d21271..784d894e20 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "regexp" "runtime" "slices" @@ -24,6 +25,7 @@ import ( "unicode/utf8" "github.com/google/uuid" + "gopkg.in/yaml.v3" "github.com/databricks/cli/acceptance/internal" "github.com/databricks/cli/internal/testutil" @@ -393,6 +395,18 @@ func runTest(t *testing.T, err = CopyDir(dir, tmpDir, inputs, outputs) require.NoError(t, err) + bundleConfigTarget := "databricks.yml" + if config.BundleConfigTarget != nil { + bundleConfigTarget = *config.BundleConfigTarget + } + + if bundleConfigTarget != "" { + configCreated := applyBundleConfig(t, tmpDir, config.BundleConfig, bundleConfigTarget) + if configCreated { + inputs[bundleConfigTarget] = true + } + } + timeout := config.Timeout if runtime.GOOS == "windows" { @@ -1001,3 +1015,74 @@ func prepareWheelBuildDirectory(t *testing.T, dir string) string { return latestWheel } + +// Applies BundleConfig setting to file named bundleConfigTarget and updates it in place if there were any changes. +// Returns true if new file was created. +func applyBundleConfig(t *testing.T, tmpDir string, bundleConfig map[string]any, bundleConfigTarget string) bool { + validConfig := make(map[string]map[string]any, len(bundleConfig)) + + for _, configName := range utils.SortedKeys(bundleConfig) { + configValue := bundleConfig[configName] + // Setting BundleConfig. to empty string disables it. + // This is useful when parent directory defines some config that child test wants to cancel. + if configValue == "" { + continue + } + cfg, ok := configValue.(map[string]any) + if !ok { + t.Fatalf("Unexpected type for BundleConfig.%s: %#v", configName, configValue) + } + validConfig[configName] = cfg + } + + if len(validConfig) == 0 { + return false + } + + configPath := filepath.Join(tmpDir, bundleConfigTarget) + configData, configExists := tryReading(t, configPath) + + newConfigData := configData + var applied []string + + for _, configName := range utils.SortedKeys(validConfig) { + configValue := validConfig[configName] + updated, err := internal.MergeBundleConfig(newConfigData, configValue) + if err != nil { + t.Fatalf("Failed to merge BundleConfig.%s: %s\nvvalue: %#v\ntext:\n%s", configName, err, configValue, newConfigData) + } + if isSameYAMLContent(newConfigData, updated) { + t.Logf("No effective updates from BundleConfig.%s", configName) + } else { + newConfigData = updated + applied = append(applied, configName) + } + } + + if newConfigData != configData { + t.Logf("Writing updated bundle config to %s. BundleConfig sections: %s", bundleConfigTarget, strings.Join(applied, ", ")) + testutil.WriteFile(t, configPath, newConfigData) + return !configExists + } + + return false +} + +// Returns true if both strings are deep-equal after unmarshalling +func isSameYAMLContent(str1, str2 string) bool { + var obj1, obj2 any + + if str1 == str2 { + return true + } + + if err := yaml.Unmarshal([]byte(str1), &obj1); err != nil { + return false + } + + if err := yaml.Unmarshal([]byte(str2), &obj2); err != nil { + return false + } + + return reflect.DeepEqual(obj1, obj2) +} diff --git a/acceptance/internal/bundle_config.go b/acceptance/internal/bundle_config.go new file mode 100644 index 0000000000..7f4e8de42b --- /dev/null +++ b/acceptance/internal/bundle_config.go @@ -0,0 +1,38 @@ +package internal + +import ( + "bytes" + + "dario.cat/mergo" + "gopkg.in/yaml.v3" +) + +func MergeBundleConfig(source string, bundleConfig map[string]any) (string, error) { + config := make(map[string]any) + + err := yaml.Unmarshal([]byte(source), &config) + if err != nil { + return "", err + } + + err = mergo.Merge( + &config, + bundleConfig, + mergo.WithoutDereference, + ) + if err != nil { + return "", err + } + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + + err = enc.Encode(config) + if err != nil { + return "", err + } + + updated := buf.String() + return updated, nil +} diff --git a/acceptance/internal/bundle_config_test.go b/acceptance/internal/bundle_config_test.go new file mode 100644 index 0000000000..2f01641cbf --- /dev/null +++ b/acceptance/internal/bundle_config_test.go @@ -0,0 +1,162 @@ +package internal + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestMergeBundleConfig(t *testing.T) { + tests := []struct { + name string + initialYaml string + bundleConfig map[string]any + expected map[string]any + }{ + { + name: "new_file", + initialYaml: "", + bundleConfig: map[string]any{}, + expected: map[string]any{}, + }, + { + name: "empty_config", + initialYaml: "{}", + bundleConfig: map[string]any{}, + expected: map[string]any{}, + }, + { + name: "simple_top_level", + initialYaml: "{}", + bundleConfig: map[string]any{ + "name": "test-bundle", + }, + expected: map[string]any{ + "name": "test-bundle", + }, + }, + { + name: "simple_top_level_new_file", + initialYaml: "", + bundleConfig: map[string]any{ + "name": "test-bundle", + }, + expected: map[string]any{ + "name": "test-bundle", + }, + }, + { + name: "nested_config", + initialYaml: "{}", + bundleConfig: map[string]any{ + "bundle": map[string]any{ + "name": "test-bundle", + }, + }, + expected: map[string]any{ + "bundle": map[string]any{ + "name": "test-bundle", + }, + }, + }, + { + name: "nested_config_new_file", + initialYaml: "", + bundleConfig: map[string]any{ + "bundle": map[string]any{ + "name": "test-bundle", + }, + }, + expected: map[string]any{ + "bundle": map[string]any{ + "name": "test-bundle", + }, + }, + }, + { + name: "merge_with_existing_config", + initialYaml: ` +bundle: + name: original-name + target: dev +`, + bundleConfig: map[string]any{ + "bundle": map[string]any{ + "name": "default-name", + }, + }, + expected: map[string]any{ + "bundle": map[string]any{ + "name": "original-name", + "target": "dev", + }, + }, + }, + { + name: "merge_with_existing_config_2", + initialYaml: `resources: {}`, + bundleConfig: map[string]any{ + "bundle": map[string]any{ + "name": "new-name", + }, + }, + expected: map[string]any{ + "bundle": map[string]any{ + "name": "new-name", + }, + "resources": map[string]any{}, + }, + }, + + { + name: "multiple_nested_levels", + initialYaml: ` +resources: + jobs: + myjob: + hello: world + pipelines: + mypipeline: + name: 123 +`, + bundleConfig: map[string]any{ + "resources": map[string]any{ + "jobs": map[string]any{ + "myjob": map[string]any{ + "name": "My Job", + }, + }, + }, + }, + expected: map[string]any{ + "resources": map[string]any{ + "jobs": map[string]any{ + "myjob": map[string]any{ + "name": "My Job", + "hello": "world", + }, + }, + "pipelines": map[string]any{ + "mypipeline": map[string]any{ + "name": 123, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := MergeBundleConfig(tt.initialYaml, tt.bundleConfig) + assert.NoError(t, err) + + var result map[string]any + require.NoError(t, yaml.Unmarshal([]byte(out), &result)) + + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index f8638ac31c..5c78ec0bb0 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -106,6 +106,27 @@ type TestConfig struct { // For cloud tests, max(Timeout, TimeoutCloud) is used for timeout // For cloud+windows tests, max(Timeout, TimeoutWindows, TimeoutCloud) is used for timeout TimeoutCloud time.Duration + + // Maps a name (arbitrary string, can be used to override/disable setting in a child config) to a mapping that specifies how + // to update databricks.yml (or other file designated by BundleConfigTarget). + // Example: + // BundleConfig.header.bundle.name = "test-bundle" + // This overwrite bundle.name in the databricks.yml, so you can omit adding """ + // bundle: + // name: somename + // """ to every test. + // If child config wants to disable or override this, they can simply do + // BundleConfig.header = "" + BundleConfig map[string]any + + // Target config for BundleConfig updates. Empty string disables BundleConfig updates. + // Null means "databricks.yml" + BundleConfigTarget *string + + // To be added: + // BundleConfigMatrix is to BundleConfig what EnvMatrix is to Env + // It creates different tests for each possible configuration update. + // BundleConfigMatrix map[string][]any } type ServerStub struct { @@ -192,8 +213,11 @@ func DoLoadConfig(t *testing.T, path string) TestConfig { require.NoError(t, err, "Failed to parse config %s", path) keys := meta.Undecoded() - if len(keys) > 0 { - t.Fatalf("Undecoded keys in %s: %#v", path, keys) + for ind, key := range keys { + if len(key) > 0 && key[0] == "BundleConfig" { + continue + } + t.Errorf("Undecoded key in %s[%d]: %#v", path, ind, key) } return config diff --git a/acceptance/selftest/bundleconfig/different_target/out.my.yml b/acceptance/selftest/bundleconfig/different_target/out.my.yml new file mode 100644 index 0000000000..a6f2be5744 --- /dev/null +++ b/acceptance/selftest/bundleconfig/different_target/out.my.yml @@ -0,0 +1,16 @@ +bundle: + name: my-bundle +resources: + jobs: + example_job: + list2: 123 + name: Example Job + new_list: + - abc + - def + new_map: + key: value + new_string: hello + string2: + - item1 + - item2 diff --git a/acceptance/selftest/bundleconfig/different_target/output.txt b/acceptance/selftest/bundleconfig/different_target/output.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/selftest/bundleconfig/different_target/script b/acceptance/selftest/bundleconfig/different_target/script new file mode 100644 index 0000000000..389111898c --- /dev/null +++ b/acceptance/selftest/bundleconfig/different_target/script @@ -0,0 +1 @@ +mv my.yml out.my.yml diff --git a/acceptance/selftest/bundleconfig/different_target/test.toml b/acceptance/selftest/bundleconfig/different_target/test.toml new file mode 100644 index 0000000000..e30696ac25 --- /dev/null +++ b/acceptance/selftest/bundleconfig/different_target/test.toml @@ -0,0 +1 @@ +BundleConfigTarget = "my.yml" diff --git a/acceptance/selftest/bundleconfig/disabled1/databricks.yml b/acceptance/selftest/bundleconfig/disabled1/databricks.yml new file mode 100644 index 0000000000..fd75daccda --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled1/databricks.yml @@ -0,0 +1 @@ +# should not be changed diff --git a/acceptance/selftest/bundleconfig/disabled1/output.txt b/acceptance/selftest/bundleconfig/disabled1/output.txt new file mode 100644 index 0000000000..fd75daccda --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled1/output.txt @@ -0,0 +1 @@ +# should not be changed diff --git a/acceptance/selftest/bundleconfig/disabled1/script b/acceptance/selftest/bundleconfig/disabled1/script new file mode 100644 index 0000000000..e17a1a94b6 --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled1/script @@ -0,0 +1 @@ +cat databricks.yml diff --git a/acceptance/selftest/bundleconfig/disabled1/test.toml b/acceptance/selftest/bundleconfig/disabled1/test.toml new file mode 100644 index 0000000000..e07ec4a780 --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled1/test.toml @@ -0,0 +1 @@ +BundleConfigTarget = "" diff --git a/acceptance/selftest/bundleconfig/disabled2/databricks.yml b/acceptance/selftest/bundleconfig/disabled2/databricks.yml new file mode 100644 index 0000000000..fd75daccda --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled2/databricks.yml @@ -0,0 +1 @@ +# should not be changed diff --git a/acceptance/selftest/bundleconfig/disabled2/output.txt b/acceptance/selftest/bundleconfig/disabled2/output.txt new file mode 100644 index 0000000000..fd75daccda --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled2/output.txt @@ -0,0 +1 @@ +# should not be changed diff --git a/acceptance/selftest/bundleconfig/disabled2/script b/acceptance/selftest/bundleconfig/disabled2/script new file mode 100644 index 0000000000..e17a1a94b6 --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled2/script @@ -0,0 +1 @@ +cat databricks.yml diff --git a/acceptance/selftest/bundleconfig/disabled2/test.toml b/acceptance/selftest/bundleconfig/disabled2/test.toml new file mode 100644 index 0000000000..c2cb6d29c8 --- /dev/null +++ b/acceptance/selftest/bundleconfig/disabled2/test.toml @@ -0,0 +1,2 @@ +BundleConfig.config1 = "" +BundleConfig.config2 = "" diff --git a/acceptance/selftest/bundleconfig/empty/output.txt b/acceptance/selftest/bundleconfig/empty/output.txt new file mode 100644 index 0000000000..a6f2be5744 --- /dev/null +++ b/acceptance/selftest/bundleconfig/empty/output.txt @@ -0,0 +1,16 @@ +bundle: + name: my-bundle +resources: + jobs: + example_job: + list2: 123 + name: Example Job + new_list: + - abc + - def + new_map: + key: value + new_string: hello + string2: + - item1 + - item2 diff --git a/acceptance/selftest/bundleconfig/empty/script b/acceptance/selftest/bundleconfig/empty/script new file mode 100644 index 0000000000..e17a1a94b6 --- /dev/null +++ b/acceptance/selftest/bundleconfig/empty/script @@ -0,0 +1 @@ +cat databricks.yml diff --git a/acceptance/selftest/bundleconfig/matching/databricks.yml b/acceptance/selftest/bundleconfig/matching/databricks.yml new file mode 100644 index 0000000000..69ba774dae --- /dev/null +++ b/acceptance/selftest/bundleconfig/matching/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: "my-bundle-2" + +# This comment should remain in the result, because BundleConfig values have no effect + +resources: + jobs: + example_job: + list2: 123456 + name: Example Job 2 + new_list: + - abc 2 + - def 2 + new_map: + key: value 2 + new_string: hello 2 + string2: + - item1 2 + - item2 2 diff --git a/acceptance/selftest/bundleconfig/matching/output.txt b/acceptance/selftest/bundleconfig/matching/output.txt new file mode 100644 index 0000000000..69ba774dae --- /dev/null +++ b/acceptance/selftest/bundleconfig/matching/output.txt @@ -0,0 +1,19 @@ +bundle: + name: "my-bundle-2" + +# This comment should remain in the result, because BundleConfig values have no effect + +resources: + jobs: + example_job: + list2: 123456 + name: Example Job 2 + new_list: + - abc 2 + - def 2 + new_map: + key: value 2 + new_string: hello 2 + string2: + - item1 2 + - item2 2 diff --git a/acceptance/selftest/bundleconfig/matching/script b/acceptance/selftest/bundleconfig/matching/script new file mode 100644 index 0000000000..e17a1a94b6 --- /dev/null +++ b/acceptance/selftest/bundleconfig/matching/script @@ -0,0 +1 @@ +cat databricks.yml diff --git a/acceptance/selftest/bundleconfig/override/databricks.yml b/acceptance/selftest/bundleconfig/override/databricks.yml new file mode 100644 index 0000000000..19e4395095 --- /dev/null +++ b/acceptance/selftest/bundleconfig/override/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: this-name-takes-priority + +resources: + jobs: + example_job: + other: attribute + string_dq: "[string]" + string_sq: '[string]' + string1: "string" + string2: "string" + list1: [string1, string2] + list2: [string1, string2] + true: true + false: false + null: null diff --git a/acceptance/selftest/bundleconfig/override/output.txt b/acceptance/selftest/bundleconfig/override/output.txt new file mode 100644 index 0000000000..5e8404824b --- /dev/null +++ b/acceptance/selftest/bundleconfig/override/output.txt @@ -0,0 +1,26 @@ +bundle: + name: this-name-takes-priority +resources: + jobs: + example_job: + false: false + true: true + null: null + list1: + - string1 + - string2 + list2: + - string1 + - string2 + name: Example Job + new_list: + - abc + - def + new_map: + key: value + new_string: hello + other: attribute + string_dq: '[string]' + string_sq: '[string]' + string1: string + string2: string diff --git a/acceptance/selftest/bundleconfig/override/script b/acceptance/selftest/bundleconfig/override/script new file mode 100644 index 0000000000..e17a1a94b6 --- /dev/null +++ b/acceptance/selftest/bundleconfig/override/script @@ -0,0 +1 @@ +cat databricks.yml diff --git a/acceptance/selftest/bundleconfig/test.toml b/acceptance/selftest/bundleconfig/test.toml new file mode 100644 index 0000000000..85b8622371 --- /dev/null +++ b/acceptance/selftest/bundleconfig/test.toml @@ -0,0 +1,12 @@ +[BundleConfig.config1.bundle] +name = "my-bundle" + +[BundleConfig.config2.resources.jobs.example_job] +name = "Example Job" +new_string = "hello" +new_list = ["abc", "def"] +new_map = {key= "value"} + +# change datatype +list2 = 123 # insert number instead of a list +string2 = ["item1", "item2"]