diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 81fe8e1344f..b42c5d5bb5a 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -18,18 +18,12 @@ limitations under the License. package impl import ( - "context" - "fmt" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/gitextractor/models" "github.com/apache/incubator-devlake/plugins/gitextractor/parser" - "github.com/apache/incubator-devlake/plugins/gitextractor/store" "github.com/apache/incubator-devlake/plugins/gitextractor/tasks" - "strings" ) var _ interface { @@ -55,6 +49,7 @@ func (p GitExtractor) Name() string { // return all available subtasks, framework will run them for you in order func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ + tasks.CloneGitRepoMeta, tasks.CollectGitCommitMeta, tasks.CollectGitBranchMeta, tasks.CollectGitTagMeta, @@ -62,7 +57,7 @@ func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta { } } -// based on task context and user input options, return data that shared among all subtasks +// PrepareTaskData based on task context and user input options, return data that shared among all subtasks func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { var op tasks.GitExtractorOptions if err := helper.Decode(options, &op, nil); err != nil { @@ -71,12 +66,10 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st if err := op.Valid(); err != nil { return nil, err } - storage := store.NewDatabase(taskCtx, op.RepoId) - repo, err := NewGitRepo(taskCtx.GetContext(), taskCtx.GetLogger(), storage, op) - if err != nil { - return nil, err + taskData := &tasks.GitExtractorTaskData{ + Options: &op, } - return repo, nil + return taskData, nil } func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error { @@ -91,20 +84,3 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error { func (p GitExtractor) RootPkgPath() string { return "github.com/apache/incubator-devlake/plugins/gitextractor" } - -// NewGitRepo create and return a new parser git repo -func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store, op tasks.GitExtractorOptions) (*parser.GitRepo, errors.Error) { - var err errors.Error - var repo *parser.GitRepo - p := parser.NewGitRepoCreator(storage, logger) - if strings.HasPrefix(op.Url, "http") { - repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User, op.Password, op.Proxy) - } else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") { - repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey, op.Passphrase) - } else if strings.HasPrefix(op.Url, "/") { - repo, err = p.LocalRepo(op.Url, op.RepoId) - } else { - return nil, errors.BadInput.New(fmt.Sprintf("unsupported url [%s]", op.Url)) - } - return repo, err -} diff --git a/backend/plugins/gitextractor/main.go b/backend/plugins/gitextractor/main.go index f1fb468d208..0224b0c97e1 100644 --- a/backend/plugins/gitextractor/main.go +++ b/backend/plugins/gitextractor/main.go @@ -78,7 +78,7 @@ func main() { "git extractor", nil, ) - repo, err := impl.NewGitRepo(subTaskCtx.GetContext(), logger, storage, tasks.GitExtractorOptions{ + repo, err := tasks.NewGitRepo(subTaskCtx.GetContext(), logger, storage, &tasks.GitExtractorOptions{ RepoId: *id, Url: *url, User: *user, diff --git a/backend/plugins/gitextractor/tasks/clone.go b/backend/plugins/gitextractor/tasks/clone.go new file mode 100644 index 00000000000..88c7694034e --- /dev/null +++ b/backend/plugins/gitextractor/tasks/clone.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "context" + "fmt" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/gitextractor/models" + "github.com/apache/incubator-devlake/plugins/gitextractor/parser" + "github.com/apache/incubator-devlake/plugins/gitextractor/store" + "strings" +) + +var CloneGitRepoMeta = plugin.SubTaskMeta{ + Name: "cloneGitRepo", + EntryPoint: CloneGitRepo, + EnabledByDefault: true, + Required: true, + Description: "clone a git repo, make it available to later tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, +} + +func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { + taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData) + if !ok { + panic("git repo reference not found on context") + } + op := taskData.Options + storage := store.NewDatabase(subTaskCtx, op.RepoId) + repo, err := NewGitRepo(subTaskCtx.GetContext(), subTaskCtx.GetLogger(), storage, op) + if err != nil { + return err + } + taskData.GitRepo = repo + subTaskCtx.TaskContext().SetData(taskData) + return nil +} + +// NewGitRepo create and return a new parser git repo +func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store, op *GitExtractorOptions) (*parser.GitRepo, errors.Error) { + var err errors.Error + var repo *parser.GitRepo + p := parser.NewGitRepoCreator(storage, logger) + if strings.HasPrefix(op.Url, "http") { + repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User, op.Password, op.Proxy) + } else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") { + repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey, op.Passphrase) + } else if strings.HasPrefix(op.Url, "/") { + repo, err = p.LocalRepo(op.Url, op.RepoId) + } else { + return nil, errors.BadInput.New(fmt.Sprintf("unsupported url [%s]", op.Url)) + } + return repo, err +} diff --git a/backend/plugins/gitextractor/tasks/git_repo_collector.go b/backend/plugins/gitextractor/tasks/git_repo_collector.go index 45090f760bc..d5132fa6c28 100644 --- a/backend/plugins/gitextractor/tasks/git_repo_collector.go +++ b/backend/plugins/gitextractor/tasks/git_repo_collector.go @@ -25,9 +25,14 @@ import ( "github.com/apache/incubator-devlake/plugins/gitextractor/parser" ) +type GitExtractorTaskData struct { + Options *GitExtractorOptions + GitRepo *parser.GitRepo +} + type GitExtractorOptions struct { RepoId string `json:"repoId"` - Name string `jsno:"name"` + Name string `json:"name"` Url string `json:"url"` User string `json:"user"` Password string `json:"password"` @@ -99,11 +104,14 @@ func CollectGitDiffLines(subTaskCtx plugin.SubTaskContext) errors.Error { } func getGitRepo(subTaskCtx plugin.SubTaskContext) *parser.GitRepo { - repo, ok := subTaskCtx.GetData().(*parser.GitRepo) + taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData) if !ok { panic("git repo reference not found on context") } - return repo + if taskData.GitRepo == nil { + panic("git repo is empty, please check subtask: clone repo") + } + return taskData.GitRepo } var CollectGitCommitMeta = plugin.SubTaskMeta{ @@ -112,6 +120,7 @@ var CollectGitCommitMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "collect git commits into Domain Layer Tables", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta}, } var CollectGitBranchMeta = plugin.SubTaskMeta{ @@ -120,6 +129,7 @@ var CollectGitBranchMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "collect git branch into Domain Layer Tables", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta}, } var CollectGitTagMeta = plugin.SubTaskMeta{ @@ -128,6 +138,7 @@ var CollectGitTagMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "collect git tag into Domain Layer Tables", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta}, } var CollectGitDiffLineMeta = plugin.SubTaskMeta{ @@ -136,4 +147,5 @@ var CollectGitDiffLineMeta = plugin.SubTaskMeta{ EnabledByDefault: false, Description: "collect git commit diff line into Domain Layer Tables", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta}, }