From 5d569fda392c1eff4cd7aaf9fb5bb31400e48c61 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 May 2025 14:51:06 +0200 Subject: [PATCH 1/3] Emit warnings and metrics when path fallback is used --- acceptance/bundle/paths/fallback/output.txt | 8 +++++ .../paths/fallback_metric/databricks.yml | 18 +++++++++++ .../bundle/paths/fallback_metric/output.txt | 31 +++++++++++++++++++ .../paths/fallback_metric/resources/job.yml | 18 +++++++++++ .../bundle/paths/fallback_metric/script | 5 +++ .../paths/fallback_metric/src/notebook.py | 2 ++ .../bundle/paths/fallback_metric/test.toml | 6 ++++ bundle/config/mutator/translate_paths_jobs.go | 4 ++- .../mutator/translate_paths_pipelines.go | 4 ++- 9 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/paths/fallback_metric/databricks.yml create mode 100644 acceptance/bundle/paths/fallback_metric/output.txt create mode 100644 acceptance/bundle/paths/fallback_metric/resources/job.yml create mode 100644 acceptance/bundle/paths/fallback_metric/script create mode 100644 acceptance/bundle/paths/fallback_metric/src/notebook.py create mode 100644 acceptance/bundle/paths/fallback_metric/test.toml diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt index e5b3e7f8f4..ddcf1af4b4 100644 --- a/acceptance/bundle/paths/fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -1,5 +1,13 @@ >>> [CLI] bundle validate -t development -o json +Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/file.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/dbt_project is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/file2.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. >>> [CLI] bundle validate -t error Error: notebook "resources/this value is overridden" not found. Local notebook references are expected diff --git a/acceptance/bundle/paths/fallback_metric/databricks.yml b/acceptance/bundle/paths/fallback_metric/databricks.yml new file mode 100644 index 0000000000..3655dc1d2d --- /dev/null +++ b/acceptance/bundle/paths/fallback_metric/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: fallback_metric + +include: + - "resources/*.yml" + +targets: + fallback: + resources: + jobs: + my_job: + tasks: + - task_key: notebook_example + notebook_task: + notebook_path: ../src/notebook.py + + default: + default: true diff --git a/acceptance/bundle/paths/fallback_metric/output.txt b/acceptance/bundle/paths/fallback_metric/output.txt new file mode 100644 index 0000000000..8cd90ac8ef --- /dev/null +++ b/acceptance/bundle/paths/fallback_metric/output.txt @@ -0,0 +1,31 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/fallback_metric/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle deploy -t fallback +Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/fallback_metric/fallback/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> cat out.requests.txt +[ + { + "key": "python_wheel_wrapper_is_set", + "value": false + } +] +[ + { + "key": "is_job_path_fallback", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + } +] diff --git a/acceptance/bundle/paths/fallback_metric/resources/job.yml b/acceptance/bundle/paths/fallback_metric/resources/job.yml new file mode 100644 index 0000000000..93e1d5fe4e --- /dev/null +++ b/acceptance/bundle/paths/fallback_metric/resources/job.yml @@ -0,0 +1,18 @@ +resources: + jobs: + my_job: + name: "placeholder" + tasks: + - task_key: notebook_example + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode + notebook_task: + notebook_path: "../src/notebook.py" diff --git a/acceptance/bundle/paths/fallback_metric/script b/acceptance/bundle/paths/fallback_metric/script new file mode 100644 index 0000000000..6fbe46ab57 --- /dev/null +++ b/acceptance/bundle/paths/fallback_metric/script @@ -0,0 +1,5 @@ +errcode trace $CLI bundle deploy +errcode trace $CLI bundle deploy -t fallback +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values' + +rm -rf out.requests.txt diff --git a/acceptance/bundle/paths/fallback_metric/src/notebook.py b/acceptance/bundle/paths/fallback_metric/src/notebook.py new file mode 100644 index 0000000000..38d86b79c7 --- /dev/null +++ b/acceptance/bundle/paths/fallback_metric/src/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Hello, World!") diff --git a/acceptance/bundle/paths/fallback_metric/test.toml b/acceptance/bundle/paths/fallback_metric/test.toml new file mode 100644 index 0000000000..b49119d206 --- /dev/null +++ b/acceptance/bundle/paths/fallback_metric/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true + +Ignore = [ + ".databricks" +] diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 2804406a01..ab99cb9fe1 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" ) func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { @@ -65,7 +66,8 @@ func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value originalValue := dyn.NewValue(originalPath, v.Locations()) nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) if nerr == nil { - // TODO: Emit a warning that this path should be rewritten. + t.b.Metrics.AddBoolValue("is_job_path_fallback", true) + log.Warnf(ctx, "path %s is defined relative to the %s directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release.", originalPath, fallback[key]) return nv, nil } } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index 313ed7018b..46139e4e44 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" ) func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { @@ -50,7 +51,8 @@ func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn. originalValue := dyn.NewValue(originalPath, v.Locations()) nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) if nerr == nil { - // TODO: Emit a warning that this path should be rewritten. + t.b.Metrics.AddBoolValue("is_pipeline_path_fallback", true) + log.Warnf(ctx, "path %s is defined relative to the %s directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release.", originalPath, fallback[key]) return nv, nil } } From 8b99d7a9ada240f3c02522f1e1605f4516843f4c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 May 2025 15:56:46 +0200 Subject: [PATCH 2/3] fix location --- acceptance/bundle/paths/fallback/output.txt | 16 ++++++++-------- .../bundle/paths/fallback_metric/output.txt | 2 +- bundle/config/mutator/translate_paths_jobs.go | 2 +- .../config/mutator/translate_paths_pipelines.go | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt index ddcf1af4b4..ee6d2c4acc 100644 --- a/acceptance/bundle/paths/fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -1,13 +1,13 @@ >>> [CLI] bundle validate -t development -o json -Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../src/file.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../src/dbt_project is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. -Warn: path ../src/file2.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:9:32). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../src/file.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:13:30). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../src/dbt_project is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:17:36). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:24:25). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:33:24). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:41:24). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:13:23). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Warn: path ../src/file2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:10:23). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. >>> [CLI] bundle validate -t error Error: notebook "resources/this value is overridden" not found. Local notebook references are expected diff --git a/acceptance/bundle/paths/fallback_metric/output.txt b/acceptance/bundle/paths/fallback_metric/output.txt index 8cd90ac8ef..7071e05350 100644 --- a/acceptance/bundle/paths/fallback_metric/output.txt +++ b/acceptance/bundle/paths/fallback_metric/output.txt @@ -6,7 +6,7 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deploy -t fallback -Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release. +Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/databricks.yml:15:32). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/fallback_metric/fallback/files... Deploying resources... Updating deployment state... diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index ab99cb9fe1..854c13fbca 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -67,7 +67,7 @@ func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) if nerr == nil { t.b.Metrics.AddBoolValue("is_job_path_fallback", true) - log.Warnf(ctx, "path %s is defined relative to the %s directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release.", originalPath, fallback[key]) + log.Warnf(ctx, "path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.", originalPath, fallback[key], v.Location()) return nv, nil } } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index 46139e4e44..b65146f796 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -52,7 +52,7 @@ func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn. nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) if nerr == nil { t.b.Metrics.AddBoolValue("is_pipeline_path_fallback", true) - log.Warnf(ctx, "path %s is defined relative to the %s directory. Please update the path to be relative to the file where it is used. This behaviour will be removed in the next release.", originalPath, fallback[key]) + log.Warnf(ctx, "path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.", originalPath, fallback[key], v.Location()) return nv, nil } } From c25e78ed911bb432072bdc4469fbe6e4590a3bd0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 May 2025 16:10:28 +0200 Subject: [PATCH 3/3] windows repls --- acceptance/bundle/paths/fallback/test.toml | 3 +++ acceptance/bundle/paths/fallback_metric/test.toml | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 acceptance/bundle/paths/fallback/test.toml diff --git a/acceptance/bundle/paths/fallback/test.toml b/acceptance/bundle/paths/fallback/test.toml new file mode 100644 index 0000000000..a298217f21 --- /dev/null +++ b/acceptance/bundle/paths/fallback/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/paths/fallback_metric/test.toml b/acceptance/bundle/paths/fallback_metric/test.toml index b49119d206..cfd0d852bc 100644 --- a/acceptance/bundle/paths/fallback_metric/test.toml +++ b/acceptance/bundle/paths/fallback_metric/test.toml @@ -4,3 +4,7 @@ RecordRequests = true Ignore = [ ".databricks" ] + +[[Repls]] +Old = '\\' +New = '/'