diff --git a/acceptance/bundle/artifacts/build_and_files_whl/databricks.yml b/acceptance/bundle/artifacts/build_and_files_whl/databricks.yml index ac47b1f0f28..529ca26bcf4 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/databricks.yml +++ b/acceptance/bundle/artifacts/build_and_files_whl/databricks.yml @@ -1,7 +1,5 @@ artifacts: artifact_with_custom_dist: - # commenting out 'type: whl' here actually makes 'bundle deploy' to build and upload correct wheel, - # because for non-whl types it does respect 'files' section. type: whl build: python setup.py bdist_wheel --dist-dir mydist files: diff --git a/acceptance/bundle/artifacts/build_and_files_whl/output.txt b/acceptance/bundle/artifacts/build_and_files_whl/output.txt index e3ed5dea706..b618de6b89a 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/output.txt +++ b/acceptance/bundle/artifacts/build_and_files_whl/output.txt @@ -10,7 +10,7 @@ Validation OK! >>> errcode [CLI] bundle deploy Building artifact_with_custom_dist... -Error: cannot find built wheel in [TEST_TMP_DIR] for package artifact_with_custom_dist - - -Exit code: 1 +Uploading mydist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Deployment complete! diff --git a/acceptance/bundle/artifacts/build_and_files_whl/test.toml b/acceptance/bundle/artifacts/build_and_files_whl/test.toml index a05cfa9087d..a030353d571 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/test.toml +++ b/acceptance/bundle/artifacts/build_and_files_whl/test.toml @@ -1,2 +1 @@ -Badness = "artifacts of type 'whl' hard-code dist directory even if explicit files section is present" RecordRequests = false diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index cdeb34df974..bad3f6f7441 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -14,6 +14,11 @@ "my_test_code": { "build": "python setup.py bdist_wheel", "dynamic_version": true, + "files": [ + { + "source": "[TEST_TMP_DIR]/my_test_code/dist/*.whl" + } + ], "path": "[TEST_TMP_DIR]/my_test_code", "type": "whl" } diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json index e510a811c67..b0f1a4b7166 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json @@ -2,6 +2,11 @@ "artifacts": { "python_artifact": { "build": "python3 setup.py bdist_wheel", + "files": [ + { + "source": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]/dist/*.whl" + } + ], "path": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]", "type": "whl" } diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 82a8ad745e9..445f45007a4 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -54,25 +54,32 @@ Resources: >>> diff.py ../out.validate.dev.json ../out.summary.dev.json --- ../out.validate.dev.json +++ ../out.summary.dev.json -@@ -45,4 +45,5 @@ +@@ -5,4 +5,6 @@ + "files": [ + { ++ "patched": "", ++ "remote_path": "", + "source": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]/dist/*.whl" + } +@@ -50,4 +52,5 @@ "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "id": "[NUMID]", "job_clusters": [ { -@@ -55,5 +56,4 @@ +@@ -60,5 +63,4 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 0, "spark_version": "15.4.x-scala2.12" } -@@ -62,5 +62,4 @@ +@@ -67,5 +69,4 @@ "max_concurrent_runs": 4, "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", - "permissions": [], "queue": { "enabled": true -@@ -113,5 +112,6 @@ +@@ -118,5 +119,6 @@ "unit": "DAYS" } - } @@ -80,13 +87,13 @@ Resources: + "url": "[DATABRICKS_URL]/jobs/[NUMID]" } }, -@@ -128,4 +128,5 @@ +@@ -133,4 +135,5 @@ "development": true, "edition": "ADVANCED", + "id": "[UUID]", "libraries": [ { -@@ -136,6 +137,6 @@ +@@ -141,6 +144,6 @@ ], "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline", - "permissions": [], @@ -95,7 +102,7 @@ Resources: + "url": "[DATABRICKS_URL]/pipelines/[UUID]" } } -@@ -146,5 +147,4 @@ +@@ -151,5 +154,4 @@ ] }, - "targets": null, @@ -127,7 +134,7 @@ Validation OK! >>> diff.py ../out.validate.dev.json ../out.validate.prod.json --- ../out.validate.dev.json +++ ../out.validate.prod.json -@@ -8,16 +8,11 @@ +@@ -13,16 +13,11 @@ }, "bundle": { - "deployment": { @@ -147,7 +154,7 @@ Validation OK! + "target": "prod", "terraform": {}, "uuid": "[UUID]" -@@ -27,13 +22,10 @@ +@@ -32,13 +27,10 @@ "resources/project_name_[UNIQUE_NAME].pipeline.yml" ], - "presets": { @@ -167,14 +174,14 @@ Validation OK! + ], "resources": { "jobs": { -@@ -41,5 +33,5 @@ +@@ -46,5 +38,5 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", -@@ -60,12 +52,9 @@ +@@ -65,12 +57,9 @@ } ], - "max_concurrent_runs": 4, @@ -189,21 +196,21 @@ Validation OK! - "dev": "[USERNAME]" }, "tasks": [ -@@ -73,5 +62,5 @@ +@@ -78,5 +67,5 @@ "job_cluster_key": "job_cluster", "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" }, "task_key": "notebook_task" -@@ -108,5 +97,5 @@ +@@ -113,5 +102,5 @@ ], "trigger": { - "pause_status": "PAUSED", + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -120,22 +109,21 @@ +@@ -125,22 +114,21 @@ "channel": "CURRENT", "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" @@ -231,7 +238,7 @@ Validation OK! + "schema": "project_name_[UNIQUE_NAME]_prod" } } -@@ -148,10 +136,10 @@ +@@ -153,10 +141,10 @@ "targets": null, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", @@ -278,7 +285,7 @@ Resources: >>> diff.py ../out.summary.dev.json ../out.summary.prod.json --- ../out.summary.dev.json +++ ../out.summary.prod.json -@@ -9,15 +9,13 @@ +@@ -16,15 +16,13 @@ "bundle": { "deployment": { - "lock": { @@ -298,7 +305,7 @@ Resources: + "target": "prod", "terraform": {}, "uuid": "[UUID]" -@@ -27,13 +25,11 @@ +@@ -34,13 +32,11 @@ "resources/project_name_[UNIQUE_NAME].pipeline.yml" ], - "presets": { @@ -319,14 +326,14 @@ Resources: + "presets": {}, "resources": { "jobs": { -@@ -41,5 +37,5 @@ +@@ -48,5 +44,5 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", -@@ -60,11 +56,8 @@ +@@ -67,11 +63,8 @@ } ], - "max_concurrent_runs": 4, @@ -340,21 +347,21 @@ Resources: - "dev": "[USERNAME]" }, "tasks": [ -@@ -72,5 +65,5 @@ +@@ -79,5 +72,5 @@ "job_cluster_key": "job_cluster", "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" }, "task_key": "notebook_task" -@@ -107,5 +100,5 @@ +@@ -114,5 +107,5 @@ ], "trigger": { - "pause_status": "PAUSED", + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -120,11 +113,10 @@ +@@ -127,11 +120,10 @@ "channel": "CURRENT", "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" @@ -368,7 +375,7 @@ Resources: - "development": true, "edition": "ADVANCED", "id": "[UUID]", -@@ -132,10 +124,10 @@ +@@ -139,10 +131,10 @@ { "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/dlt_pipeline" @@ -382,7 +389,7 @@ Resources: + "schema": "project_name_[UNIQUE_NAME]_prod", "url": "[DATABRICKS_URL]/pipelines/[UUID]" } -@@ -148,10 +140,10 @@ +@@ -155,10 +147,10 @@ }, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 3c54f3037a1..ff30e10fbd6 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/libs/exec" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/patchwheel" - "github.com/databricks/cli/libs/python" "github.com/databricks/cli/libs/utils" ) @@ -51,23 +50,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { break } - if a.Type == "whl" { - dir := a.Path - distPath := filepath.Join(a.Path, "dist") - wheels := python.FindFilesWithSuffixInPath(distPath, ".whl") - if len(wheels) == 0 { - diags = diags.Extend(diag.Errorf("cannot find built wheel in %s for package %s", dir, artifactName)) - break - } - for _, wheel := range wheels { - a.Files = append(a.Files, config.ArtifactFile{ - Source: wheel, - }) - } - } else { - log.Warnf(ctx, "%s: Build succeeded", artifactName) - } - // We need to expand glob reference after build mutator is applied because // if we do it before, any files that are generated by build command will // not be included into artifact.Files and thus will not be uploaded. diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index c2a2d67a1c1..2bd0a8ba245 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -42,6 +42,15 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics if artifact.BuildCommand == "" && len(artifact.Files) == 0 { artifact.BuildCommand = python.GetExecutable() + " setup.py bdist_wheel" } + + // Wheel builds write to `./dist`. Pick up all wheel files by default if nothing is specified. + if len(artifact.Files) == 0 { + artifact.Files = []config.ArtifactFile{ + { + Source: filepath.Join(artifact.Path, "dist", "*.whl"), + }, + } + } } l := b.Config.GetLocation("artifacts." + artifactName)