Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions acceptance/bundle/paths/fallback/output.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@

>>> [CLI] bundle validate -t development -o json
Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:9:32). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../src/file.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:13:30). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../src/dbt_project is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:17:36). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:24:25). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:33:24). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:41:24). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:13:23). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Warn: path ../src/file2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:10:23). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.

>>> [CLI] bundle validate -t error
Error: notebook "resources/this value is overridden" not found. Local notebook references are expected
Expand Down
3 changes: 3 additions & 0 deletions acceptance/bundle/paths/fallback/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[[Repls]]
Old = '\\'
New = '/'
18 changes: 18 additions & 0 deletions acceptance/bundle/paths/fallback_metric/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
bundle:
name: fallback_metric

include:
- "resources/*.yml"

targets:
fallback:
resources:
jobs:
my_job:
tasks:
- task_key: notebook_example
notebook_task:
notebook_path: ../src/notebook.py

default:
default: true
31 changes: 31 additions & 0 deletions acceptance/bundle/paths/fallback_metric/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/fallback_metric/default/files...
Deploying resources...
Updating deployment state...
Deployment complete!

>>> [CLI] bundle deploy -t fallback
Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/databricks.yml:15:32). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/fallback_metric/fallback/files...
Deploying resources...
Updating deployment state...
Deployment complete!

>>> cat out.requests.txt
[
{
"key": "python_wheel_wrapper_is_set",
"value": false
}
]
[
{
"key": "is_job_path_fallback",
"value": true
},
{
"key": "python_wheel_wrapper_is_set",
"value": false
}
]
18 changes: 18 additions & 0 deletions acceptance/bundle/paths/fallback_metric/resources/job.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
resources:
jobs:
my_job:
name: "placeholder"
tasks:
- task_key: notebook_example
new_cluster:
spark_version: 15.4.x-scala2.12
node_type_id: i3.xlarge
data_security_mode: SINGLE_USER
num_workers: 0
spark_conf:
spark.master: "local[*, 4]"
spark.databricks.cluster.profile: singleNode
custom_tags:
ResourceClass: SingleNode
notebook_task:
notebook_path: "../src/notebook.py"
5 changes: 5 additions & 0 deletions acceptance/bundle/paths/fallback_metric/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
errcode trace $CLI bundle deploy
errcode trace $CLI bundle deploy -t fallback
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values'

rm -rf out.requests.txt
2 changes: 2 additions & 0 deletions acceptance/bundle/paths/fallback_metric/src/notebook.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Databricks notebook source
print("Hello, World!")
10 changes: 10 additions & 0 deletions acceptance/bundle/paths/fallback_metric/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Cloud = false
RecordRequests = true

Ignore = [
".databricks"
]

[[Repls]]
Old = '\\'
New = '/'
4 changes: 3 additions & 1 deletion bundle/config/mutator/translate_paths_jobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/bundle/config/mutator/paths"

"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/log"
)

func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
Expand Down Expand Up @@ -65,7 +66,8 @@ func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value
originalValue := dyn.NewValue(originalPath, v.Locations())
nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts)
if nerr == nil {
// TODO: Emit a warning that this path should be rewritten.
t.b.Metrics.AddBoolValue("is_job_path_fallback", true)
log.Warnf(ctx, "path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.", originalPath, fallback[key], v.Location())
return nv, nil
}
}
Expand Down
4 changes: 3 additions & 1 deletion bundle/config/mutator/translate_paths_pipelines.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/databricks/cli/bundle/config/mutator/paths"

"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/log"
)

func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) {
Expand Down Expand Up @@ -50,7 +51,8 @@ func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn.
originalValue := dyn.NewValue(originalPath, v.Locations())
nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts)
if nerr == nil {
// TODO: Emit a warning that this path should be rewritten.
t.b.Metrics.AddBoolValue("is_pipeline_path_fallback", true)
log.Warnf(ctx, "path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.", originalPath, fallback[key], v.Location())
return nv, nil
}
}
Expand Down
Loading