-
Notifications
You must be signed in to change notification settings - Fork 168
Add fs mkdirs command for dbfs #432
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
24 commits
Select commit
Hold shift + click to select a range
b5f5f11
Add fs ls command for dbfs
shreyas-goenka a4879c0
added integration tests
shreyas-goenka 2706beb
lint
shreyas-goenka 6b667b5
added test for ls on flie
shreyas-goenka db953ac
Merge remote-tracking branch 'origin' into fs-ls
shreyas-goenka 492382c
added resolvedbfs path func
shreyas-goenka 3dc4770
manually check for dbfs prefix
shreyas-goenka f14ceba
Add fs mkdirs command for dbfs
shreyas-goenka 19a2051
comments
shreyas-goenka 8050dc5
move sort outside
shreyas-goenka fc7fbbe
-
shreyas-goenka b048c86
address comments
shreyas-goenka 265e517
address comments 2
shreyas-goenka 0882609
added preallocation of size
shreyas-goenka 9160a6e
initialize dbfs client at root
shreyas-goenka bdc3fca
merge
shreyas-goenka 7bdaa00
-
shreyas-goenka a24f0b0
Merge remote-tracking branch 'origin' into fs-mkdirs
shreyas-goenka bc2411f
-
shreyas-goenka d717d9a
-
shreyas-goenka c2efe08
add allias
shreyas-goenka b63abeb
Merge remote-tracking branch 'origin' into fs-mkdirs
shreyas-goenka ef2d8e2
add comment
shreyas-goenka c187b0e
Merge branch 'main' into fs-mkdirs
shreyas-goenka File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| package fs | ||
|
|
||
| import ( | ||
| "github.com/databricks/cli/cmd/root" | ||
| "github.com/databricks/cli/libs/filer" | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| var mkdirCmd = &cobra.Command{ | ||
| Use: "mkdir DIR_PATH", | ||
| // Alias `mkdirs` for this command exists for legacy purposes. This command | ||
| // is called databricks fs mkdirs in our legacy CLI: https://github.com/databricks/databricks-cli | ||
| Aliases: []string{"mkdirs"}, | ||
| Short: "Make directories", | ||
| Long: `Mkdir will create directories along the path to the argument directory.`, | ||
| Args: cobra.ExactArgs(1), | ||
| PreRunE: root.MustWorkspaceClient, | ||
|
|
||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| w := root.WorkspaceClient(ctx) | ||
|
|
||
| path, err := resolveDbfsPath(args[0]) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| f, err := filer.NewDbfsClient(w, "/") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return f.Mkdir(ctx, path) | ||
| }, | ||
| } | ||
|
|
||
| func init() { | ||
| fsCmd.AddCommand(mkdirCmd) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| package internal | ||
|
|
||
| import ( | ||
| "context" | ||
| "path" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/databricks/cli/libs/filer" | ||
| "github.com/databricks/databricks-sdk-go" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TesFsMkdirCreatesDirectory(t *testing.T) { | ||
| t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) | ||
|
|
||
| ctx := context.Background() | ||
| w, err := databricks.NewWorkspaceClient() | ||
| require.NoError(t, err) | ||
|
|
||
| tmpDir := temporaryDbfsDir(t, w) | ||
|
|
||
| f, err := filer.NewDbfsClient(w, tmpDir) | ||
| require.NoError(t, err) | ||
|
shreyas-goenka marked this conversation as resolved.
|
||
|
|
||
| // create directory "a" | ||
| stdout, stderr := RequireSuccessfulRun(t, "fs", "mkdir", "dbfs:"+path.Join(tmpDir, "a")) | ||
| assert.Equal(t, "", stderr.String()) | ||
| assert.Equal(t, "", stdout.String()) | ||
|
|
||
| // assert directory "a" is created | ||
| info, err := f.Stat(ctx, "a") | ||
| require.NoError(t, err) | ||
| assert.Equal(t, "a", info.Name()) | ||
| assert.Equal(t, true, info.IsDir()) | ||
| } | ||
|
|
||
| func TestFsMkdirCreatesMultipleDirectories(t *testing.T) { | ||
| t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) | ||
|
|
||
| ctx := context.Background() | ||
| w, err := databricks.NewWorkspaceClient() | ||
| require.NoError(t, err) | ||
|
|
||
| tmpDir := temporaryDbfsDir(t, w) | ||
|
|
||
| f, err := filer.NewDbfsClient(w, tmpDir) | ||
| require.NoError(t, err) | ||
|
|
||
| // create directory /a/b/c | ||
| stdout, stderr := RequireSuccessfulRun(t, "fs", "mkdir", "dbfs:"+path.Join(tmpDir, "a", "b", "c")) | ||
| assert.Equal(t, "", stderr.String()) | ||
| assert.Equal(t, "", stdout.String()) | ||
|
|
||
| // assert directory "a" is created | ||
| infoA, err := f.Stat(ctx, "a") | ||
| require.NoError(t, err) | ||
| assert.Equal(t, "a", infoA.Name()) | ||
| assert.Equal(t, true, infoA.IsDir()) | ||
|
|
||
| // assert directory "b" is created | ||
| infoB, err := f.Stat(ctx, "a/b") | ||
| require.NoError(t, err) | ||
| assert.Equal(t, "b", infoB.Name()) | ||
| assert.Equal(t, true, infoB.IsDir()) | ||
|
|
||
| // assert directory "c" is created | ||
| infoC, err := f.Stat(ctx, "a/b/c") | ||
| require.NoError(t, err) | ||
| assert.Equal(t, "c", infoC.Name()) | ||
| assert.Equal(t, true, infoC.IsDir()) | ||
| } | ||
|
|
||
| func TestFsMkdirWhenDirectoryAlreadyExists(t *testing.T) { | ||
| t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) | ||
|
|
||
| ctx := context.Background() | ||
| w, err := databricks.NewWorkspaceClient() | ||
| require.NoError(t, err) | ||
|
|
||
| tmpDir := temporaryDbfsDir(t, w) | ||
|
|
||
| // create directory "a" | ||
| f, err := filer.NewDbfsClient(w, tmpDir) | ||
| require.NoError(t, err) | ||
| err = f.Mkdir(ctx, "a") | ||
| require.NoError(t, err) | ||
|
|
||
| // assert run is successful without any errors | ||
| stdout, stderr := RequireSuccessfulRun(t, "fs", "mkdir", "dbfs:"+path.Join(tmpDir, "a")) | ||
| assert.Equal(t, "", stderr.String()) | ||
| assert.Equal(t, "", stdout.String()) | ||
| } | ||
|
|
||
| func TestFsMkdirWhenFileExistsAtPath(t *testing.T) { | ||
| t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) | ||
|
|
||
| ctx := context.Background() | ||
| w, err := databricks.NewWorkspaceClient() | ||
| require.NoError(t, err) | ||
|
|
||
| tmpDir := temporaryDbfsDir(t, w) | ||
|
|
||
| // create file hello | ||
| f, err := filer.NewDbfsClient(w, tmpDir) | ||
| require.NoError(t, err) | ||
| err = f.Write(ctx, "hello", strings.NewReader("abc")) | ||
| require.NoError(t, err) | ||
|
|
||
| // assert run fails | ||
| _, _, err = RequireErrorRun(t, "fs", "mkdir", "dbfs:"+path.Join(tmpDir, "hello")) | ||
| assert.ErrorContains(t, err, "Cannot create directory") | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.