From 9a04a3a41c0f9023335fdfc8f30e9935a22da111 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 15 Jun 2023 13:56:35 -0700 Subject: [PATCH 1/4] add prefix Signed-off-by: Yee Hing Tong --- dataproxy/service.go | 8 +++++++- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dataproxy/service.go b/dataproxy/service.go index a62d37b43..b5e56c9cc 100644 --- a/dataproxy/service.go +++ b/dataproxy/service.go @@ -79,8 +79,14 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp md5 := base64.StdEncoding.EncodeToString(req.ContentMd5) urlSafeMd5 := base32.StdEncoding.EncodeToString(req.ContentMd5) + var prefix string + if req.FilenameRoot != "" { + prefix = req.FilenameRoot + } else { + prefix = urlSafeMd5 + } storagePath, err := createStorageLocation(ctx, s.dataStore, s.cfg.Upload, - req.Project, req.Domain, urlSafeMd5, req.Filename) + req.Project, req.Domain, prefix, req.Filename) if err != nil { return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to create shardedStorageLocation, Error: %v", err) } diff --git a/go.mod b/go.mod index 46ef4b0de..e42402486 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cloudevents/sdk-go/v2 v2.8.0 github.com/coreos/go-oidc v2.2.1+incompatible github.com/evanphx/json-patch v4.12.0+incompatible - github.com/flyteorg/flyteidl v1.5.7 + github.com/flyteorg/flyteidl v1.5.11-0.20230615182642-01c711f0a4d7 github.com/flyteorg/flyteplugins v1.0.56 github.com/flyteorg/flytepropeller v1.1.87 github.com/flyteorg/flytestdlib v1.0.15 diff --git a/go.sum b/go.sum index 255fa8893..86c6bbbc6 100644 --- a/go.sum +++ b/go.sum @@ -314,6 +314,8 @@ github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8S github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flyteorg/flyteidl v1.5.7 h1:voAxMMFsKOseNFSlCyRGlpegqtQXtJjyxgsQzZg4tts= github.com/flyteorg/flyteidl v1.5.7/go.mod h1:EtE/muM2lHHgBabjYcxqe9TWeJSL0kXwbI0RgVwI4Og= +github.com/flyteorg/flyteidl v1.5.11-0.20230615182642-01c711f0a4d7 h1:lDqtfcevqPPwcCwHxoUpFs0JiQ7U1nEMMrMNy8p8Iko= +github.com/flyteorg/flyteidl v1.5.11-0.20230615182642-01c711f0a4d7/go.mod h1:EtE/muM2lHHgBabjYcxqe9TWeJSL0kXwbI0RgVwI4Og= github.com/flyteorg/flyteplugins v1.0.56 h1:kBTDgTpdSi7wcptk4cMwz5vfh1MU82VaUMMboe1InXw= github.com/flyteorg/flyteplugins v1.0.56/go.mod h1:aFCKSn8TPzxSAILIiogHtUnHlUCN9+y6Vf+r9R4KZDU= github.com/flyteorg/flytepropeller v1.1.87 h1:Px7ASDjrWyeVrUb15qXmhw9QK7xPcFjL5Yetr2P6iGM= From 5f85815697730f686d13f56d739e49d31f27c322 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 19 Jun 2023 09:35:43 -0700 Subject: [PATCH 2/4] save a variable Signed-off-by: Yee Hing Tong --- dataproxy/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataproxy/service.go b/dataproxy/service.go index b5e56c9cc..60596245e 100644 --- a/dataproxy/service.go +++ b/dataproxy/service.go @@ -77,13 +77,13 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp } md5 := base64.StdEncoding.EncodeToString(req.ContentMd5) - urlSafeMd5 := base32.StdEncoding.EncodeToString(req.ContentMd5) var prefix string if req.FilenameRoot != "" { prefix = req.FilenameRoot } else { - prefix = urlSafeMd5 + // url safe base32 encoding + prefix = base32.StdEncoding.EncodeToString(req.ContentMd5) } storagePath, err := createStorageLocation(ctx, s.dataStore, s.cfg.Upload, req.Project, req.Domain, prefix, req.Filename) From 369a68c067805a172c20a458525950dea707b553 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 19 Jun 2023 09:56:04 -0700 Subject: [PATCH 3/4] Update dataproxy/service.go Co-authored-by: Haytham Abuelfutuh Signed-off-by: Yee Hing Tong --- dataproxy/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataproxy/service.go b/dataproxy/service.go index 60596245e..05b364daf 100644 --- a/dataproxy/service.go +++ b/dataproxy/service.go @@ -79,7 +79,7 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp md5 := base64.StdEncoding.EncodeToString(req.ContentMd5) var prefix string - if req.FilenameRoot != "" { + if len(req.FilenameRoot) > 0 { prefix = req.FilenameRoot } else { // url safe base32 encoding From b8c246a44ac62858752962ac6ea3109d397db2a3 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 22 Jun 2023 12:47:57 -0700 Subject: [PATCH 4/4] use released idl Signed-off-by: Yee Hing Tong --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e42402486..388c650f3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cloudevents/sdk-go/v2 v2.8.0 github.com/coreos/go-oidc v2.2.1+incompatible github.com/evanphx/json-patch v4.12.0+incompatible - github.com/flyteorg/flyteidl v1.5.11-0.20230615182642-01c711f0a4d7 + github.com/flyteorg/flyteidl v1.5.11 github.com/flyteorg/flyteplugins v1.0.56 github.com/flyteorg/flytepropeller v1.1.87 github.com/flyteorg/flytestdlib v1.0.15 diff --git a/go.sum b/go.sum index 86c6bbbc6..dedb61115 100644 --- a/go.sum +++ b/go.sum @@ -316,6 +316,8 @@ github.com/flyteorg/flyteidl v1.5.7 h1:voAxMMFsKOseNFSlCyRGlpegqtQXtJjyxgsQzZg4t github.com/flyteorg/flyteidl v1.5.7/go.mod h1:EtE/muM2lHHgBabjYcxqe9TWeJSL0kXwbI0RgVwI4Og= github.com/flyteorg/flyteidl v1.5.11-0.20230615182642-01c711f0a4d7 h1:lDqtfcevqPPwcCwHxoUpFs0JiQ7U1nEMMrMNy8p8Iko= github.com/flyteorg/flyteidl v1.5.11-0.20230615182642-01c711f0a4d7/go.mod h1:EtE/muM2lHHgBabjYcxqe9TWeJSL0kXwbI0RgVwI4Og= +github.com/flyteorg/flyteidl v1.5.11 h1:Xcb17YqNstl+dHQsK+o0Ac+1l1U154wXivg28O3C5l0= +github.com/flyteorg/flyteidl v1.5.11/go.mod h1:EtE/muM2lHHgBabjYcxqe9TWeJSL0kXwbI0RgVwI4Og= github.com/flyteorg/flyteplugins v1.0.56 h1:kBTDgTpdSi7wcptk4cMwz5vfh1MU82VaUMMboe1InXw= github.com/flyteorg/flyteplugins v1.0.56/go.mod h1:aFCKSn8TPzxSAILIiogHtUnHlUCN9+y6Vf+r9R4KZDU= github.com/flyteorg/flytepropeller v1.1.87 h1:Px7ASDjrWyeVrUb15qXmhw9QK7xPcFjL5Yetr2P6iGM=