Use filer in repofiles struct

internal/filer_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/databricks-sdk-go/service/files"
-	"github.com/databricks/databricks-sdk-go/service/workspace"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -159,35 +158,17 @@ func runFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	assert.Len(t, entries, 1)
 	assert.Equal(t, "c", entries[0].Name())
 	assert.True(t, entries[0].IsDir())
-}
 
-func temporaryWorkspaceDir(t *testing.T, w *databricks.WorkspaceClient) string {
-	ctx := context.Background()
-	me, err := w.CurrentUser.Me(ctx)
-	require.NoError(t, err)
+	// Expect an error trying to call ReadDir on a file
+	_, err = f.ReadDir(ctx, "/hello.txt")
+	assert.ErrorIs(t, err, filer.ErrNotADirectory)
 
-	path := fmt.Sprintf("/Users/%s/%s", me.UserName, RandomName("integration-test-filer-wsfs-"))
-
-	// Ensure directory exists, but doesn't exist YET!
-	// Otherwise we could inadvertently remove a directory that already exists on cleanup.
-	t.Logf("mkdir %s", path)
-	err = w.Workspace.MkdirsByPath(ctx, path)
+	// Expect 0 entries for an empty directory
+	err = f.Mkdir(ctx, "empty-dir")
 	require.NoError(t, err)
-
-	// Remove test directory on test completion.
-	t.Cleanup(func() {
-		t.Logf("rm -rf %s", path)
-		err := w.Workspace.Delete(ctx, workspace.Delete{
-			Path:      path,
-			Recursive: true,
-		})
-		if err == nil || apierr.IsMissing(err) {
-			return
-		}
-		t.Logf("unable to remove temporary workspace directory %s: %#v", path, err)
-	})
-
-	return path
+	entries, err = f.ReadDir(ctx, "empty-dir")
+	assert.NoError(t, err)
+	assert.Len(t, entries, 0)
 }
 
 func setupWorkspaceFilesTest(t *testing.T) (context.Context, filer.Filer) {

internal/helpers.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -14,6 +15,11 @@ import (
 
 	"github.com/databricks/cli/cmd/root"
 	_ "github.com/databricks/cli/cmd/version"
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	_ "github.com/databricks/cli/cmd/workspace"
@@ -176,3 +182,52 @@ func writeFile(t *testing.T, name string, body string) string {
 	f.Close()
 	return f.Name()
 }
+
+func temporaryWorkspaceDir(t *testing.T, w *databricks.WorkspaceClient) string {
+	ctx := context.Background()
+	me, err := w.CurrentUser.Me(ctx)
+	require.NoError(t, err)
+
+	path := fmt.Sprintf("/Users/%s/%s", me.UserName, RandomName("integration-test-wsfs-"))
+
+	// Ensure directory exists, but doesn't exist YET!
+	// Otherwise we could inadvertently remove a directory that already exists on cleanup.
+	t.Logf("mkdir %s", path)
+	err = w.Workspace.MkdirsByPath(ctx, path)
+	require.NoError(t, err)
+
+	// Remove test directory on test completion.
+	t.Cleanup(func() {
+		t.Logf("rm -rf %s", path)
+		err := w.Workspace.Delete(ctx, workspace.Delete{
+			Path:      path,
+			Recursive: true,
+		})
+		if err == nil || apierr.IsMissing(err) {
+			return
+		}
+		t.Logf("unable to remove temporary workspace directory %s: %#v", path, err)
+	})
+
+	return path
+}
+
+func assertFileContains(t *testing.T, ctx context.Context, f filer.Filer, name, contents string) {
+	r, err := f.Read(ctx, name)
+	require.NoError(t, err)
+
+	var b bytes.Buffer
+	_, err = io.Copy(&b, r)
+	require.NoError(t, err)
+
+	assert.Contains(t, b.String(), contents)
+}
+
+func assertNotebookExists(t *testing.T, ctx context.Context, w *databricks.WorkspaceClient, path string) {
+	info, err := w.Workspace.ListAll(ctx, workspace.ListWorkspaceRequest{
+		Path: path,
+	})
+	require.NoError(t, err)
+	assert.Len(t, info, 1)
+	assert.Equal(t, info[0].ObjectType, workspace.ObjectTypeNotebook)
+}

internal/repofiles_test.go

@@ -0,0 +1,251 @@
+package internal
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/cli/libs/sync/repofiles"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type repofilesTestHelper struct {
+	w   *databricks.WorkspaceClient
+	f   filer.Filer
+	ctx context.Context
+	t   *testing.T
+
+	localRoot  string
+	remoteRoot string
+}
+
+func setupRepofilesTestHelper(t *testing.T, ctx context.Context) *repofilesTestHelper {
+	// t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+
+	w, err := databricks.NewWorkspaceClient()
+	require.NoError(t, err)
+
+	// initialize client
+	wsfsTmpDir := temporaryWorkspaceDir(t, w)
+	localTmpDir := t.TempDir()
+
+	require.NoError(t, err)
+	f, err := filer.NewWorkspaceFilesClient(w, wsfsTmpDir)
+	require.NoError(t, err)
+
+	return &repofilesTestHelper{
+		w:   w,
+		f:   f,
+		ctx: ctx,
+		t:   t,
+
+		localRoot:  localTmpDir,
+		remoteRoot: wsfsTmpDir,
+	}
+}
+
+func (h *repofilesTestHelper) createLocalFile(name string, content string) {
+	absPath := filepath.Join(h.localRoot, name)
+	err := os.MkdirAll(filepath.Dir(absPath), os.ModePerm)
+	require.NoError(h.t, err)
+	err = os.WriteFile(absPath, []byte(content), os.ModePerm)
+	require.NoError(h.t, err)
+}
+
+func (h *repofilesTestHelper) createRemoteFile(name string, content string) {
+	h.f.Write(h.ctx, name, strings.NewReader(content), filer.CreateParentDirectories)
+}
+
+func (h *repofilesTestHelper) createRemoteDirectory(name string) {
+	h.f.Mkdir(h.ctx, name)
+}
+
+func (h *repofilesTestHelper) assertRemoteFileContent(name string, content string) {
+	assertFileContains(h.t, h.ctx, h.f, name, content)
+}
+
+func (h *repofilesTestHelper) assertRemoteFileType(name string, fileType workspace.ObjectType) {
+	info, err := h.f.Stat(h.ctx, name)
+	require.NoError(h.t, err)
+
+	objectInfo := info.Sys().(workspace.ObjectInfo)
+	assert.Equal(h.t, fileType, objectInfo.ObjectType)
+}
+
+func TestRepoFilesPutFile(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: true,
+	})
+	require.NoError(t, err)
+
+	// create local file
+	helper.createLocalFile("foo.txt", "hello, world")
+	err = r.PutFile(ctx, "foo.txt")
+	require.NoError(t, err)
+
+	// Expect PUT to succeed
+	helper.assertRemoteFileContent("foo.txt", "hello, world")
+}
+
+func TestRepoFilesPutFileOverwritesNotebook(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: true,
+	})
+	require.NoError(t, err)
+
+	// Create notebook in workspace
+	helper.createRemoteFile("foo.py", "#Databricks notebook source\nprint(1)")
+	helper.assertRemoteFileType("foo", workspace.ObjectTypeNotebook)
+
+	// Put file and assert file PUT succeeded
+	helper.createLocalFile("foo", "this file will overwrite the notebook")
+	err = r.PutFile(ctx, "foo")
+	assert.NoError(t, err)
+	helper.assertRemoteFileContent("foo", "this file will overwrite the notebook")
+	helper.assertRemoteFileType("foo", workspace.ObjectTypeFile)
+}
+
+func TestRepoFilesPutFileOverwritesEmptyDirectoryTree(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: true,
+	})
+	require.NoError(t, err)
+
+	// create empty remote directory tree
+	helper.createRemoteDirectory("foo/a/b/c")
+	helper.createRemoteDirectory("foo/a/b/d/e")
+	helper.createRemoteDirectory("foo/f/g/i")
+
+	// assert directory tree is created
+	helper.assertRemoteFileType("foo", workspace.ObjectTypeDirectory)
+	helper.assertRemoteFileType("foo/a/b/c", workspace.ObjectTypeDirectory)
+	helper.assertRemoteFileType("foo/f/g/i", workspace.ObjectTypeDirectory)
+	helper.assertRemoteFileType("foo/a/b/d/e", workspace.ObjectTypeDirectory)
+
+	// Create local file and PUT it into the workspace
+	helper.createLocalFile("foo", "hello, world")
+	err = r.PutFile(ctx, "foo")
+	require.NoError(t, err)
+	helper.assertRemoteFileContent("foo", "hello, world")
+	helper.assertRemoteFileType("foo", workspace.ObjectTypeFile)
+}
+
+func TestRepoFilesPutFileInDirOverwritesExistingNotebook(t *testing.T) {
+	// TODO: Skipping this test for now since the workspace-files import API has a
+	// bug and does not return the error message we need
+	t.SkipNow()
+
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: true,
+	})
+	require.NoError(t, err)
+
+	// create remote notebook
+	helper.createRemoteFile("foo.py", "#Databricks notebook source\nprint(1)")
+	helper.assertRemoteFileType("foo", workspace.ObjectTypeNotebook)
+
+	// create local file and PUT it in the workspace
+	helper.createLocalFile("foo/hello.txt", "just a file")
+	err = r.PutFile(ctx, "foo/hello.txt")
+	require.NoError(t, err)
+
+	// Assert PUT succeeeded
+	helper.assertRemoteFileType("foo", workspace.ObjectTypeDirectory)
+	helper.assertRemoteFileContent("foo/bar.txt", "just a file")
+}
+
+func TestRepoFilesPutFileWithoutOverwrite(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: false,
+	})
+	require.NoError(t, err)
+
+	// create local file
+	helper.createLocalFile("foo.txt", "hello, world")
+	err = r.PutFile(ctx, "foo.txt")
+	require.NoError(t, err)
+
+	// Expect PUT to succeed
+	helper.assertRemoteFileContent("foo.txt", "hello, world")
+}
+
+func TestRepoFilesPutFileWithoutOverwriteFails(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: false,
+	})
+	require.NoError(t, err)
+
+	// create remote file
+	helper.createRemoteFile("foo.txt", "this file already exists in the workspace")
+
+	// create local file
+	helper.createLocalFile("foo.txt", "this file will attempt to overwrite the workspace file and fail")
+
+	// assert overwrite fails
+	err = r.PutFile(ctx, "foo.txt")
+	assert.ErrorIs(t, err, fs.ErrExist)
+}
+
+func TestRepoFilesPutFileWithoutOverwriteFailsIfDirectoryExists(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: false,
+	})
+	require.NoError(t, err)
+
+	helper.createRemoteDirectory("foo")
+
+	// create local file
+	helper.createLocalFile("foo", "hello, world")
+	err = r.PutFile(ctx, "foo")
+
+	// Assert PUT failed because file already exists
+	assert.ErrorIs(t, err, fs.ErrExist)
+}
+
+func TestRepoFilesPutFileWithoutOverwriteFailsIfNotebookExists(t *testing.T) {
+	ctx := context.Background()
+	helper := setupRepofilesTestHelper(t, ctx)
+
+	r, err := repofiles.Create(helper.remoteRoot, helper.localRoot, helper.w, &repofiles.RepoFileOptions{
+		OverwriteIfExists: false,
+	})
+	require.NoError(t, err)
+
+	// create remote notebook
+	helper.createRemoteFile("foo.py", "#Databricks notebook source\nprint(1)")
+
+	// create local file
+	helper.createLocalFile("foo", "hello, world")
+	err = r.PutFile(ctx, "foo")
+
+	// Assert PUT failed because file already exists
+	assert.ErrorIs(t, err, fs.ErrExist)
+}

libs/filer/dbfs_client.go

@@ -59,7 +59,7 @@ func (info dbfsFileInfo) IsDir() bool {
 }
 
 func (info dbfsFileInfo) Sys() any {
-	return nil
+	return info.fi
 }
 
 // DbfsClient implements the [Filer] interface for the DBFS backend.
@@ -222,6 +222,10 @@ func (w *DbfsClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, e
 		return nil, err
 	}
 
+	if len(res.Files) == 1 && res.Files[0].Path == absPath {
+		return nil, NotADirectory{absPath}
+	}
+
 	info := make([]fs.DirEntry, len(res.Files))
 	for i, v := range res.Files {
 		info[i] = dbfsDirEntry{dbfsFileInfo: dbfsFileInfo{fi: v}}