acc: add BundleConfig setting

acceptance/acceptance_test.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"runtime"
 	"slices"
@@ -24,6 +25,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/google/uuid"
+	"gopkg.in/yaml.v3"
 
 	"github.com/databricks/cli/acceptance/internal"
 	"github.com/databricks/cli/internal/testutil"
@@ -393,6 +395,18 @@ func runTest(t *testing.T,
 	err = CopyDir(dir, tmpDir, inputs, outputs)
 	require.NoError(t, err)
 
+	bundleConfigTarget := "databricks.yml"
+	if config.BundleConfigTarget != nil {
+		bundleConfigTarget = *config.BundleConfigTarget
+	}
+
+	if bundleConfigTarget != "" {
+		configCreated := applyBundleConfig(t, tmpDir, config.BundleConfig, bundleConfigTarget)
+		if configCreated {
+			inputs[bundleConfigTarget] = true
+		}
+	}
+
 	timeout := config.Timeout
 
 	if runtime.GOOS == "windows" {
@@ -1001,3 +1015,74 @@ func prepareWheelBuildDirectory(t *testing.T, dir string) string {
 
 	return latestWheel
 }
+
+// Applies BundleConfig setting to file named bundleConfigTarget and updates it in place if there were any changes.
+// Returns true if new file was created.
+func applyBundleConfig(t *testing.T, tmpDir string, bundleConfig map[string]any, bundleConfigTarget string) bool {
+	validConfig := make(map[string]map[string]any, len(bundleConfig))
+
+	for _, configName := range utils.SortedKeys(bundleConfig) {
+		configValue := bundleConfig[configName]
+		// Setting BundleConfig.<name> to empty string disables it.
+		// This is useful when parent directory defines some config that child test wants to cancel.
+		if configValue == "" {
+			continue
+		}
+		cfg, ok := configValue.(map[string]any)
+		if !ok {
+			t.Fatalf("Unexpected type for BundleConfig.%s: %#v", configName, configValue)
+		}
+		validConfig[configName] = cfg
+	}
+
+	if len(validConfig) == 0 {
+		return false
+	}
+
+	configPath := filepath.Join(tmpDir, bundleConfigTarget)
+	configData, configExists := tryReading(t, configPath)
+
+	newConfigData := configData
+	var applied []string
+
+	for _, configName := range utils.SortedKeys(validConfig) {
+		configValue := validConfig[configName]
+		updated, err := internal.MergeBundleConfig(newConfigData, configValue)
+		if err != nil {
+			t.Fatalf("Failed to merge BundleConfig.%s: %s\nvvalue: %#v\ntext:\n%s", configName, err, configValue, newConfigData)
+		}
+		if isSameYAMLContent(newConfigData, updated) {
+			t.Logf("No effective updates from BundleConfig.%s", configName)
+		} else {
+			newConfigData = updated
+			applied = append(applied, configName)
+		}
+	}
+
+	if newConfigData != configData {
+		t.Logf("Writing updated bundle config to %s. BundleConfig sections: %s", bundleConfigTarget, strings.Join(applied, ", "))
+		testutil.WriteFile(t, configPath, newConfigData)
+		return !configExists
+	}
+
+	return false
+}
+
+// Returns true if both strings are deep-equal after unmarshalling
+func isSameYAMLContent(str1, str2 string) bool {
+	var obj1, obj2 any
+
+	if str1 == str2 {
+		return true
+	}
+
+	if err := yaml.Unmarshal([]byte(str1), &obj1); err != nil {
+		return false
+	}
+
+	if err := yaml.Unmarshal([]byte(str2), &obj2); err != nil {
+		return false
+	}
+
+	return reflect.DeepEqual(obj1, obj2)
+}

acceptance/internal/bundle_config.go

@@ -0,0 +1,38 @@
+package internal
+
+import (
+	"bytes"
+
+	"dario.cat/mergo"
+	"gopkg.in/yaml.v3"
+)
+
+func MergeBundleConfig(source string, bundleConfig map[string]any) (string, error) {
+	config := make(map[string]any)
+
+	err := yaml.Unmarshal([]byte(source), &config)
+	if err != nil {
+		return "", err
+	}
+
+	err = mergo.Merge(
+		&config,
+		bundleConfig,
+		mergo.WithoutDereference,
+	)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	enc := yaml.NewEncoder(&buf)
+	enc.SetIndent(2)
+
+	err = enc.Encode(config)
+	if err != nil {
+		return "", err
+	}
+
+	updated := buf.String()
+	return updated, nil
+}

acceptance/internal/bundle_config_test.go

@@ -0,0 +1,162 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestMergeBundleConfig(t *testing.T) {
+	tests := []struct {
+		name         string
+		initialYaml  string
+		bundleConfig map[string]any
+		expected     map[string]any
+	}{
+		{
+			name:         "new_file",
+			initialYaml:  "",
+			bundleConfig: map[string]any{},
+			expected:     map[string]any{},
+		},
+		{
+			name:         "empty_config",
+			initialYaml:  "{}",
+			bundleConfig: map[string]any{},
+			expected:     map[string]any{},
+		},
+		{
+			name:        "simple_top_level",
+			initialYaml: "{}",
+			bundleConfig: map[string]any{
+				"name": "test-bundle",
+			},
+			expected: map[string]any{
+				"name": "test-bundle",
+			},
+		},
+		{
+			name:        "simple_top_level_new_file",
+			initialYaml: "",
+			bundleConfig: map[string]any{
+				"name": "test-bundle",
+			},
+			expected: map[string]any{
+				"name": "test-bundle",
+			},
+		},
+		{
+			name:        "nested_config",
+			initialYaml: "{}",
+			bundleConfig: map[string]any{
+				"bundle": map[string]any{
+					"name": "test-bundle",
+				},
+			},
+			expected: map[string]any{
+				"bundle": map[string]any{
+					"name": "test-bundle",
+				},
+			},
+		},
+		{
+			name:        "nested_config_new_file",
+			initialYaml: "",
+			bundleConfig: map[string]any{
+				"bundle": map[string]any{
+					"name": "test-bundle",
+				},
+			},
+			expected: map[string]any{
+				"bundle": map[string]any{
+					"name": "test-bundle",
+				},
+			},
+		},
+		{
+			name: "merge_with_existing_config",
+			initialYaml: `
+bundle:
+  name: original-name
+  target: dev
+`,
+			bundleConfig: map[string]any{
+				"bundle": map[string]any{
+					"name": "default-name",
+				},
+			},
+			expected: map[string]any{
+				"bundle": map[string]any{
+					"name":   "original-name",
+					"target": "dev",
+				},
+			},
+		},
+		{
+			name:        "merge_with_existing_config_2",
+			initialYaml: `resources: {}`,
+			bundleConfig: map[string]any{
+				"bundle": map[string]any{
+					"name": "new-name",
+				},
+			},
+			expected: map[string]any{
+				"bundle": map[string]any{
+					"name": "new-name",
+				},
+				"resources": map[string]any{},
+			},
+		},
+
+		{
+			name: "multiple_nested_levels",
+			initialYaml: `
+resources:
+  jobs:
+    myjob:
+      hello: world
+  pipelines:
+    mypipeline:
+      name: 123
+`,
+			bundleConfig: map[string]any{
+				"resources": map[string]any{
+					"jobs": map[string]any{
+						"myjob": map[string]any{
+							"name": "My Job",
+						},
+					},
+				},
+			},
+			expected: map[string]any{
+				"resources": map[string]any{
+					"jobs": map[string]any{
+						"myjob": map[string]any{
+							"name":  "My Job",
+							"hello": "world",
+						},
+					},
+					"pipelines": map[string]any{
+						"mypipeline": map[string]any{
+							"name": 123,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, err := MergeBundleConfig(tt.initialYaml, tt.bundleConfig)
+			assert.NoError(t, err)
+
+			var result map[string]any
+			require.NoError(t, yaml.Unmarshal([]byte(out), &result))
+
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}

acceptance/internal/config.go

@@ -106,6 +106,27 @@ type TestConfig struct {
 	// For cloud tests, max(Timeout, TimeoutCloud) is used for timeout
 	// For cloud+windows tests, max(Timeout, TimeoutWindows, TimeoutCloud) is used for timeout
 	TimeoutCloud time.Duration
+
+	// Maps a name (arbitrary string, can be used to override/disable setting in a child config) to a mapping that specifies how
+	// to update databricks.yml (or other file designated by BundleConfigTarget).
+	// Example:
+	// BundleConfig.header.bundle.name = "test-bundle"
+	// This overwrite bundle.name in the databricks.yml, so you can omit adding """
+	// bundle:
+	//   name: somename
+	// """ to every test.
+	// If child config wants to disable or override this, they can simply do
+	// BundleConfig.header = ""
+	BundleConfig map[string]any
+
+	// Target config for BundleConfig updates. Empty string disables BundleConfig updates.
+	// Null means "databricks.yml"
+	BundleConfigTarget *string
+
+	// To be added:
+	// BundleConfigMatrix is to BundleConfig what EnvMatrix is to Env
+	// It creates different tests for each possible configuration update.
+	// BundleConfigMatrix map[string][]any
 }
 
 type ServerStub struct {
@@ -192,8 +213,11 @@ func DoLoadConfig(t *testing.T, path string) TestConfig {
 	require.NoError(t, err, "Failed to parse config %s", path)
 
 	keys := meta.Undecoded()
-	if len(keys) > 0 {
-		t.Fatalf("Undecoded keys in %s: %#v", path, keys)
+	for ind, key := range keys {
+		if len(key) > 0 && key[0] == "BundleConfig" {
+			continue
+		}
+		t.Errorf("Undecoded key in %s[%d]: %#v", path, ind, key)
 	}
 
 	return config

acceptance/selftest/bundleconfig/different_target/out.my.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: my-bundle
+resources:
+  jobs:
+    example_job:
+      list2: 123
+      name: Example Job
+      new_list:
+        - abc
+        - def
+      new_map:
+        key: value
+      new_string: hello
+      string2:
+        - item1
+        - item2

acceptance/selftest/bundleconfig/different_target/script

@@ -0,0 +1 @@
+mv my.yml out.my.yml

acceptance/selftest/bundleconfig/different_target/test.toml

@@ -0,0 +1 @@
+BundleConfigTarget = "my.yml"

acceptance/selftest/bundleconfig/disabled1/databricks.yml

@@ -0,0 +1 @@
+# should not be changed

acceptance/selftest/bundleconfig/disabled1/output.txt

@@ -0,0 +1 @@
+# should not be changed

acceptance/selftest/bundleconfig/disabled1/script

@@ -0,0 +1 @@
+cat databricks.yml

acceptance/selftest/bundleconfig/disabled1/test.toml

@@ -0,0 +1 @@
+BundleConfigTarget = ""

acceptance/selftest/bundleconfig/disabled2/databricks.yml

@@ -0,0 +1 @@
+# should not be changed

acceptance/selftest/bundleconfig/disabled2/output.txt

@@ -0,0 +1 @@
+# should not be changed

acceptance/selftest/bundleconfig/disabled2/script

@@ -0,0 +1 @@
+cat databricks.yml

acceptance/selftest/bundleconfig/disabled2/test.toml

@@ -0,0 +1,2 @@
+BundleConfig.config1 = ""
+BundleConfig.config2 = ""