From 5d21ceb33c1390e949b249156c278f42574638ef Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Aug 2023 17:31:16 +0200 Subject: [PATCH 1/5] Added support for experimental scripts section --- bundle/config/experimental.go | 5 +++ bundle/config/root.go | 2 ++ bundle/config/scripts.go | 11 ++++++ bundle/phases/build.go | 4 +++ bundle/phases/deploy.go | 4 +++ bundle/scripts/scripts.go | 68 +++++++++++++++++++++++++++++++++++ 6 files changed, 94 insertions(+) create mode 100644 bundle/config/experimental.go create mode 100644 bundle/config/scripts.go create mode 100644 bundle/scripts/scripts.go diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go new file mode 100644 index 00000000000..e5a59a7cc7a --- /dev/null +++ b/bundle/config/experimental.go @@ -0,0 +1,5 @@ +package config + +type Experimental struct { + Scripts map[ScriptHook]Command `json:"scripts,omitempty"` +} diff --git a/bundle/config/root.go b/bundle/config/root.go index 52f88737881..952be9f86a4 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -74,6 +74,8 @@ type Root struct { // If not specified, the code below initializes this field with a // single default-initialized environment called "default". Environments map[string]*Environment `json:"environments,omitempty"` + + Experimental *Experimental `json:"experimental,omitempty"` } func Load(path string) (*Root, error) { diff --git a/bundle/config/scripts.go b/bundle/config/scripts.go new file mode 100644 index 00000000000..80bce3b47de --- /dev/null +++ b/bundle/config/scripts.go @@ -0,0 +1,11 @@ +package config + +type Command string +type ScriptHook string + +const ( + ScriptPreBuild ScriptHook = "prebuild" + ScriptPostBuild ScriptHook = "postbuild" + ScriptPreDeploy ScriptHook = "predeploy" + ScriptPostDeploy ScriptHook = "postdeploy" +) diff --git a/bundle/phases/build.go b/bundle/phases/build.go index fe90c3691e3..27182372c7d 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -3,7 +3,9 @@ package phases import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/interpolation" + "github.com/databricks/cli/bundle/scripts" ) // The build phase builds artifacts. @@ -13,7 +15,9 @@ func Build() bundle.Mutator { []bundle.Mutator{ artifacts.DetectPackages(), artifacts.InferMissingProperties(), + scripts.Execute(config.ScriptPreBuild), artifacts.BuildAll(), + scripts.Execute(config.ScriptPostBuild), interpolation.Interpolate( interpolation.IncludeLookupsInPath("artifacts"), ), diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 011bb4b2ba4..735ea875e49 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -3,16 +3,19 @@ package phases import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/bundle/scripts" ) // The deploy phase deploys artifacts and resources. func Deploy() bundle.Mutator { deployMutator := bundle.Seq( + scripts.Execute(config.ScriptPreDeploy), lock.Acquire(), bundle.Defer( bundle.Seq( @@ -29,6 +32,7 @@ func Deploy() bundle.Mutator { ), lock.Release(lock.GoalDeploy), ), + scripts.Execute(config.ScriptPostDeploy), ) return newPhase( diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go new file mode 100644 index 00000000000..9212cb247bc --- /dev/null +++ b/bundle/scripts/scripts.go @@ -0,0 +1,68 @@ +package scripts + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/cmdio" +) + +func Execute(hook config.ScriptHook) bundle.Mutator { + return &script{ + scriptHook: hook, + } +} + +type script struct { + scriptHook config.ScriptHook +} + +func (m *script) Name() string { + return fmt.Sprintf("scripts.%s", m.scriptHook) +} + +func (m *script) Apply(ctx context.Context, b *bundle.Bundle) error { + out, err := executeHook(ctx, b, m.scriptHook) + cmdio.LogString(ctx, bytes.NewBuffer(out).String()) + return err +} + +func executeHook(ctx context.Context, b *bundle.Bundle, hook config.ScriptHook) ([]byte, error) { + command := getCommmand(b, hook) + if command == "" { + return nil, nil + } + commands := strings.Split(strings.ReplaceAll(string(command), "\r\n", "\n"), "\n") + + out := make([][]byte, 0) + for _, command := range commands { + if command == "" { + continue + } + + subcommands := strings.Split(command, " && ") + for _, subcommand := range subcommands { + buildParts := strings.Split(subcommand, " ") + cmd := exec.CommandContext(ctx, buildParts[0], buildParts[1:]...) + res, err := cmd.CombinedOutput() + if err != nil { + return res, err + } + out = append(out, res) + } + } + return bytes.Join(out, []byte{}), nil +} + +func getCommmand(b *bundle.Bundle, hook config.ScriptHook) config.Command { + if b.Config.Experimental == nil || b.Config.Experimental.Scripts == nil { + return "" + } + + return b.Config.Experimental.Scripts[hook] +} From 17d4472bd9515d59fd6670a8866b9a9e9a6db0e1 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 12 Sep 2023 13:38:35 +0200 Subject: [PATCH 2/5] Use sh --- bundle/config/root.go | 4 ++-- bundle/scripts/scripts.go | 30 +++++++++++------------------- bundle/scripts/scripts_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 bundle/scripts/scripts_test.go diff --git a/bundle/config/root.go b/bundle/config/root.go index 4cbb1db343e..465d8a62e18 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -84,8 +84,8 @@ type Root struct { // RunAs section allows to define an execution identity for jobs and pipelines runs RunAs *jobs.JobRunAs `json:"run_as,omitempty"` - - Experimental *Experimental `json:"experimental,omitempty"` + + Experimental *Experimental `json:"experimental,omitempty"` } func Load(path string) (*Root, error) { diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 9212cb247bc..852a6818cd3 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "os/exec" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -37,26 +36,14 @@ func executeHook(ctx context.Context, b *bundle.Bundle, hook config.ScriptHook) if command == "" { return nil, nil } - commands := strings.Split(strings.ReplaceAll(string(command), "\r\n", "\n"), "\n") - out := make([][]byte, 0) - for _, command := range commands { - if command == "" { - continue - } - - subcommands := strings.Split(command, " && ") - for _, subcommand := range subcommands { - buildParts := strings.Split(subcommand, " ") - cmd := exec.CommandContext(ctx, buildParts[0], buildParts[1:]...) - res, err := cmd.CombinedOutput() - if err != nil { - return res, err - } - out = append(out, res) - } + interpreter, err := findInterpreter() + if err != nil { + return nil, err } - return bytes.Join(out, []byte{}), nil + + cmd := exec.CommandContext(ctx, interpreter, "-c", string(command)) + return cmd.CombinedOutput() } func getCommmand(b *bundle.Bundle, hook config.ScriptHook) config.Command { @@ -66,3 +53,8 @@ func getCommmand(b *bundle.Bundle, hook config.ScriptHook) config.Command { return b.Config.Experimental.Scripts[hook] } + +func findInterpreter() (string, error) { + // At the moment we just return 'sh' on all platforms and use it to execute scripts + return "sh", nil +} diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go new file mode 100644 index 00000000000..d6f23dee599 --- /dev/null +++ b/bundle/scripts/scripts_test.go @@ -0,0 +1,27 @@ +package scripts + +import ( + "context" + "strings" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/require" +) + +func TestExecutesHook(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{ + Scripts: map[config.ScriptHook]config.Command{ + config.ScriptPreBuild: "echo 'Hello'", + }, + }, + }, + } + out, err := executeHook(context.Background(), b, config.ScriptPreBuild) + + require.NoError(t, err) + require.Equal(t, "Hello", strings.TrimSpace(string(out))) +} From 180b51bf65e2218bbdc1776e8e69c3b5bf087704 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 12 Sep 2023 14:20:53 +0200 Subject: [PATCH 3/5] streaming and preinit --- bundle/config/mutator/mutator.go | 3 ++ bundle/config/scripts.go | 1 + bundle/phases/deploy.go | 2 +- bundle/scripts/scripts.go | 47 ++++++++++++++++++++++++++------ bundle/scripts/scripts_test.go | 9 ++++-- 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index ff1f96f50e0..aa762e8e6e3 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -2,10 +2,13 @@ package mutator import ( "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/scripts" ) func DefaultMutators() []bundle.Mutator { return []bundle.Mutator{ + scripts.Execute(config.ScriptPreInit), ProcessRootIncludes(), DefineDefaultTarget(), LoadGitDetails(), diff --git a/bundle/config/scripts.go b/bundle/config/scripts.go index 80bce3b47de..eaf7e1d15ca 100644 --- a/bundle/config/scripts.go +++ b/bundle/config/scripts.go @@ -4,6 +4,7 @@ type Command string type ScriptHook string const ( + ScriptPreInit ScriptHook = "preinit" ScriptPreBuild ScriptHook = "prebuild" ScriptPostBuild ScriptHook = "postbuild" ScriptPreDeploy ScriptHook = "predeploy" diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 731b006e3ed..a8ca75186b6 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/python" + "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" ) diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 852a6818cd3..1a8a471caa5 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -1,14 +1,17 @@ package scripts import ( - "bytes" + "bufio" "context" "fmt" + "io" "os/exec" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" ) func Execute(hook config.ScriptHook) bundle.Mutator { @@ -26,24 +29,52 @@ func (m *script) Name() string { } func (m *script) Apply(ctx context.Context, b *bundle.Bundle) error { - out, err := executeHook(ctx, b, m.scriptHook) - cmdio.LogString(ctx, bytes.NewBuffer(out).String()) - return err + cmd, out, err := executeHook(ctx, b, m.scriptHook) + if err != nil { + return err + } + if cmd == nil { + log.Debugf(ctx, "No script defined for %s, skipping", m.scriptHook) + return nil + } + + cmdio.LogString(ctx, fmt.Sprintf("Executing '%s' script", m.scriptHook)) + + reader := bufio.NewReader(out) + line, err := reader.ReadString('\n') + for err == nil { + cmdio.LogString(ctx, strings.TrimSpace(line)) + line, err = reader.ReadString('\n') + } + + return cmd.Wait() } -func executeHook(ctx context.Context, b *bundle.Bundle, hook config.ScriptHook) ([]byte, error) { +func executeHook(ctx context.Context, b *bundle.Bundle, hook config.ScriptHook) (*exec.Cmd, io.Reader, error) { command := getCommmand(b, hook) if command == "" { - return nil, nil + return nil, nil, nil } interpreter, err := findInterpreter() if err != nil { - return nil, err + return nil, nil, err } cmd := exec.CommandContext(ctx, interpreter, "-c", string(command)) - return cmd.CombinedOutput() + cmd.Dir = b.Config.Path + + outPipe, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + + errPipe, err := cmd.StderrPipe() + if err != nil { + return nil, nil, err + } + + return cmd, io.MultiReader(outPipe, errPipe), cmd.Start() } func getCommmand(b *bundle.Bundle, hook config.ScriptHook) config.Command { diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go index d6f23dee599..8b7aa0d1b83 100644 --- a/bundle/scripts/scripts_test.go +++ b/bundle/scripts/scripts_test.go @@ -1,6 +1,7 @@ package scripts import ( + "bufio" "context" "strings" "testing" @@ -20,8 +21,12 @@ func TestExecutesHook(t *testing.T) { }, }, } - out, err := executeHook(context.Background(), b, config.ScriptPreBuild) + _, out, err := executeHook(context.Background(), b, config.ScriptPreBuild) + require.NoError(t, err) + + reader := bufio.NewReader(out) + line, err := reader.ReadString('\n') require.NoError(t, err) - require.Equal(t, "Hello", strings.TrimSpace(string(out))) + require.Equal(t, "Hello", strings.TrimSpace(line)) } From c5b3caf02a7f37e3e0279a1d2131668b6787abc1 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 12 Sep 2023 14:26:30 +0200 Subject: [PATCH 4/5] added post init --- bundle/config/scripts.go | 1 + bundle/phases/build.go | 2 +- bundle/phases/initialize.go | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bundle/config/scripts.go b/bundle/config/scripts.go index eaf7e1d15ca..8d6efc9fab9 100644 --- a/bundle/config/scripts.go +++ b/bundle/config/scripts.go @@ -5,6 +5,7 @@ type ScriptHook string const ( ScriptPreInit ScriptHook = "preinit" + ScriptPostInit ScriptHook = "postinit" ScriptPreBuild ScriptHook = "prebuild" ScriptPostBuild ScriptHook = "postbuild" ScriptPreDeploy ScriptHook = "predeploy" diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 27182372c7d..760967fca2c 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -13,9 +13,9 @@ func Build() bundle.Mutator { return newPhase( "build", []bundle.Mutator{ + scripts.Execute(config.ScriptPreBuild), artifacts.DetectPackages(), artifacts.InferMissingProperties(), - scripts.Execute(config.ScriptPreBuild), artifacts.BuildAll(), scripts.Execute(config.ScriptPostBuild), interpolation.Interpolate( diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 546a8478bc0..431fe27d446 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -2,10 +2,12 @@ package phases import ( "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/interpolation" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/scripts" ) // The initialize phase fills in defaults and connects to the workspace. @@ -30,6 +32,7 @@ func Initialize() bundle.Mutator { mutator.ProcessTargetMode(), mutator.TranslatePaths(), terraform.Initialize(), + scripts.Execute(config.ScriptPostInit), }, ) } From 745e861e919dc40be92e389edd593ea2326213fb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 14 Sep 2023 12:02:24 +0200 Subject: [PATCH 5/5] moved to experimental.go --- bundle/config/experimental.go | 13 +++++++++++++ bundle/config/scripts.go | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 bundle/config/scripts.go diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index e5a59a7cc7a..be0e7d8fefa 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -3,3 +3,16 @@ package config type Experimental struct { Scripts map[ScriptHook]Command `json:"scripts,omitempty"` } + +type Command string +type ScriptHook string + +// These hook names are subject to change and currently experimental +const ( + ScriptPreInit ScriptHook = "preinit" + ScriptPostInit ScriptHook = "postinit" + ScriptPreBuild ScriptHook = "prebuild" + ScriptPostBuild ScriptHook = "postbuild" + ScriptPreDeploy ScriptHook = "predeploy" + ScriptPostDeploy ScriptHook = "postdeploy" +) diff --git a/bundle/config/scripts.go b/bundle/config/scripts.go deleted file mode 100644 index 8d6efc9fab9..00000000000 --- a/bundle/config/scripts.go +++ /dev/null @@ -1,13 +0,0 @@ -package config - -type Command string -type ScriptHook string - -const ( - ScriptPreInit ScriptHook = "preinit" - ScriptPostInit ScriptHook = "postinit" - ScriptPreBuild ScriptHook = "prebuild" - ScriptPostBuild ScriptHook = "postbuild" - ScriptPreDeploy ScriptHook = "predeploy" - ScriptPostDeploy ScriptHook = "postdeploy" -)