diff --git a/cmd/root.go b/cmd/root.go index 231f40e..42a67c5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,6 +10,7 @@ import ( "github.com/jetstack/tally/internal/db" "github.com/jetstack/tally/internal/manager" "github.com/jetstack/tally/internal/output" + "github.com/jetstack/tally/internal/repositories" "github.com/jetstack/tally/internal/scorecard" "github.com/jetstack/tally/internal/types" "github.com/spf13/cobra" @@ -81,15 +82,24 @@ var rootCmd = &cobra.Command{ } defer r.Close() } - pkgs, err := bom.PackagesFromBOM(r, bom.Format(ro.Format)) + sbom, err := bom.ParseBOM(r, bom.Format(ro.Format)) + if err != nil { + return err + } + pkgs, err := sbom.Packages() if err != nil { return err } fmt.Fprintf(os.Stderr, "Found %d supported packages in BOM\n", len(pkgs)) - // Find repositories for the packages from the database + // Map packages to repositories, using various different sources + repoMapper := repositories.From( + repositories.PackageMapper, + repositories.BOMMapper(sbom), + repositories.DBMapper(tallyDB), + ) for i, pkg := range pkgs { - repos, err := tallyDB.GetRepositories(ctx, pkg.System, pkg.Name) + repos, err := repoMapper.Repositories(ctx, pkg) if err != nil { if err != db.ErrNotFound { return err diff --git a/internal/bom/bom.go b/internal/bom/bom.go index 6f7ef0c..9c84750 100644 --- a/internal/bom/bom.go +++ b/internal/bom/bom.go @@ -1,15 +1,11 @@ package bom import ( - "encoding/json" - "errors" "fmt" "io" - "strings" "github.com/CycloneDX/cyclonedx-go" "github.com/jetstack/tally/internal/types" - "github.com/package-url/packageurl-go" ) // Format is a supported SBOM format @@ -28,168 +24,26 @@ var Formats = []Format{ FormatSyftJSON, } -var ( - ErrUnsupportedPackageType = errors.New("unsupported package type") -) +// BOM is a generic representation of a BOM +type BOM interface { + // Packages retrieves all the supported packages from the BOM + Packages() ([]types.Package, error) + + // Repositories retrieves any repository information from the BOM for + // the provided package + Repositories(types.Package) ([]string, error) +} -// PackagesFromBOM extracts packages from a supported SBOM format -func PackagesFromBOM(r io.Reader, format Format) ([]types.Package, error) { +// ParseBOM parses a BOM from a given format +func ParseBOM(r io.Reader, format Format) (BOM, error) { switch format { case FormatCycloneDXJSON: - return packagesFromCycloneDX(r, cyclonedx.BOMFileFormatJSON) + return parseCycloneDX(r, cyclonedx.BOMFileFormatJSON) case FormatCycloneDXXML: - return packagesFromCycloneDX(r, cyclonedx.BOMFileFormatXML) + return parseCycloneDX(r, cyclonedx.BOMFileFormatXML) case FormatSyftJSON: - return packagesFromSyftJSON(r) + return parseSyftJSON(r) default: return nil, fmt.Errorf("unsupported format: %s", format) } } - -func packagesFromCycloneDX(r io.Reader, format cyclonedx.BOMFileFormat) ([]types.Package, error) { - cdxBOM := &cyclonedx.BOM{} - if err := cyclonedx.NewBOMDecoder(r, format).Decode(cdxBOM); err != nil { - return nil, fmt.Errorf("decoding cyclonedx BOM: %w", err) - } - var ( - pkgs []types.Package - components []cyclonedx.Component - ) - if cdxBOM.Metadata != nil && cdxBOM.Metadata.Component != nil { - components = append(components, *cdxBOM.Metadata.Component) - } - if cdxBOM.Components != nil { - components = append(components, *cdxBOM.Components...) - } - if err := walkCycloneDXComponents(components, func(component cyclonedx.Component) error { - if component.PackageURL == "" { - return nil - } - purl, err := packageurl.FromString(component.PackageURL) - if err != nil { - return err - } - pkg, err := packageFromPurl(purl) - if errors.Is(err, ErrUnsupportedPackageType) { - return nil - } - if err != nil { - return err - } - if !containsPackage(pkgs, pkg) { - pkgs = append(pkgs, pkg) - } - - return nil - - }); err != nil { - return nil, err - } - - return pkgs, nil -} - -func walkCycloneDXComponents(components []cyclonedx.Component, fn func(cyclonedx.Component) error) error { - for _, component := range components { - if err := fn(component); err != nil { - return err - } - if component.Components == nil { - continue - } - if err := walkCycloneDXComponents(*component.Components, fn); err != nil { - return err - } - } - - return nil -} - -type syftJSON struct { - Artifacts []syftArtifact `json:"artifacts"` -} - -type syftArtifact struct { - Purl string `json:"purl"` -} - -func packagesFromSyftJSON(r io.Reader) ([]types.Package, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - doc := &syftJSON{} - if err := json.Unmarshal(data, doc); err != nil { - return nil, err - } - var pkgs []types.Package - for _, a := range doc.Artifacts { - if a.Purl == "" { - continue - } - purl, err := packageurl.FromString(a.Purl) - if err != nil { - return nil, err - } - pkg, err := packageFromPurl(purl) - if errors.Is(err, ErrUnsupportedPackageType) { - continue - } - if err != nil { - return nil, err - } - pkgs = append(pkgs, pkg) - } - - return pkgs, nil -} - -func packageFromPurl(purl packageurl.PackageURL) (types.Package, error) { - switch purl.Type { - case packageurl.TypeNPM: - name := purl.Name - if purl.Namespace != "" { - name = purl.Namespace + "/" + name - } - return types.Package{ - System: "NPM", - Name: name, - }, nil - case packageurl.TypeGolang: - name := purl.Name - if purl.Namespace != "" { - name = purl.Namespace + "/" + purl.Name - } - return types.Package{ - System: "GO", - Name: name, - }, nil - case packageurl.TypeMaven: - return types.Package{ - System: "MAVEN", - Name: strings.Join([]string{purl.Namespace, purl.Name}, ":"), - }, nil - case packageurl.TypePyPi: - return types.Package{ - System: "PYPI", - Name: purl.Name, - }, nil - case "cargo": - return types.Package{ - System: "CARGO", - Name: purl.Name, - }, nil - default: - return types.Package{}, ErrUnsupportedPackageType - } -} - -func containsPackage(pkgs []types.Package, pkg types.Package) bool { - for _, p := range pkgs { - if p.Equals(pkg) { - return true - } - } - - return false -} diff --git a/internal/bom/bom_test.go b/internal/bom/bom_test.go deleted file mode 100644 index 55a46de..0000000 --- a/internal/bom/bom_test.go +++ /dev/null @@ -1,288 +0,0 @@ -package bom - -import ( - "bytes" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/jetstack/tally/internal/types" -) - -var testCycloneDXJSON = []byte(` -{ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "serialNumber": "urn:uuid:5e0841b1-88e1-4dd8-b706-77457fb3e779", - "version": 1, - "metadata": { - "component": { - "bom-ref": "1234567", - "type": "application", - "name": "foo/bar", - "version": "v0.2.5", - "purl": "pkg:golang/foo/bar@v0.2.5" - } - }, - "components": [ - { - "bom-ref": "0", - "type": "library", - "name": "HdrHistogram", - "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9" - }, - { - "bom-ref": "1", - "type": "library", - "name": "adduser", - "purl": "pkg:deb/debian/adduser@3.118?arch=all\u0026distro=debian-11" - }, - { - "bom-ref": "2", - "type": "library", - "name": "release-utils", - "purl": "pkg:golang/sigs.k8s.io/release-utils@v0.7.3" - }, - { - "bom-ref": "3", - "type": "library", - "name": "zwitch", - "purl": "pkg:npm/zwitch@2.0.2" - }, - { - "bom-ref": "4", - "type": "library", - "name": "getrandom", - "purl": "pkg:cargo/getrandom@0.2.7" - }, - { - "bom-ref": "5", - "type": "library", - "name": "barfoo" - }, - { - "bom-ref": "6", - "type": "library", - "name": "zope.interface", - "purl": "pkg:pypi/zope.interface@5.4.0" - } - ] -} -`) - -var testCycloneDXXML = []byte(` - - - - - foo/bar - v0.2.5 - pkg:golang/foo/bar@v0.2.5 - - - - - HdrHistogram - pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9 - - - adduser - pkg:deb/debian/adduser@3.118?arch=all&distro=debian-11 - - - release-utils - pkg:golang/sigs.k8s.io/release-utils@v0.7.3 - - - zwitch - pkg:npm/zwitch@2.0.2 - - - getrandom - pkg:cargo/getrandom@0.2.7 - - - barfoo - - - zope.interface - pkg:pypi/zope.interface@5.4.0 - - - -`) - -var testSyftJSON = []byte(` -{ - "artifacts": [ - { - "id": "0", - "name": "foo", - "purl": "pkg:golang/foo/bar@v0.2.5" - }, - { - "id": "1", - "name": "HdrHistogram", - "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9" - }, - { - "id": "2", - "name": "adduser", - "purl": "pkg:deb/debian/adduser@3.118?arch=all\u0026distro=debian-11" - }, - { - "id": "3", - "name": "release-utils", - "purl": "pkg:golang/sigs.k8s.io/release-utils@v0.7.3" - }, - { - "id": "4", - "name": "zwitch", - "purl": "pkg:npm/zwitch@2.0.2" - }, - { - "id": "5", - "name": "getrandom", - "purl": "pkg:cargo/getrandom@0.2.7" - }, - { - "id": "6", - "name": "barfoo" - }, - { - "id": "7", - "name": "zope.interface", - "purl": "pkg:pypi/zope.interface@5.4.0" - } - ] -} -`) - -func TestPackagesFromBOM(t *testing.T) { - testCases := map[string]struct { - data []byte - format Format - wantPkgs []types.Package - wantErr bool - }{ - "cyclonedx-json: detect expected packages": { - data: testCycloneDXJSON, - format: FormatCycloneDXJSON, - wantPkgs: []types.Package{ - { - System: "GO", - Name: "foo/bar", - }, - { - System: "MAVEN", - Name: "org.hdrhistogram:HdrHistogram", - }, - { - System: "GO", - Name: "sigs.k8s.io/release-utils", - }, - { - System: "NPM", - Name: "zwitch", - }, - { - System: "CARGO", - Name: "getrandom", - }, - { - System: "PYPI", - Name: "zope.interface", - }, - }, - }, - "cyclonedx-json: returns error when input is cyclonedx xml": { - data: testCycloneDXXML, - format: FormatCycloneDXJSON, - wantErr: true, - }, - "cyclonedx-xml: detect expected packages": { - data: testCycloneDXXML, - format: FormatCycloneDXXML, - wantPkgs: []types.Package{ - { - System: "GO", - Name: "foo/bar", - }, - { - System: "MAVEN", - Name: "org.hdrhistogram:HdrHistogram", - }, - { - System: "GO", - Name: "sigs.k8s.io/release-utils", - }, - { - System: "NPM", - Name: "zwitch", - }, - { - System: "CARGO", - Name: "getrandom", - }, - { - System: "PYPI", - Name: "zope.interface", - }, - }, - }, - "cyclonedx-xml: returns error when input is cyclonedx json": { - data: testCycloneDXJSON, - format: FormatCycloneDXXML, - wantErr: true, - }, - "syft-json: detect expected packages": { - data: testSyftJSON, - format: FormatSyftJSON, - wantPkgs: []types.Package{ - { - System: "GO", - Name: "foo/bar", - }, - { - System: "MAVEN", - Name: "org.hdrhistogram:HdrHistogram", - }, - { - System: "GO", - Name: "sigs.k8s.io/release-utils", - }, - { - System: "NPM", - Name: "zwitch", - }, - { - System: "CARGO", - Name: "getrandom", - }, - { - System: "PYPI", - Name: "zope.interface", - }, - }, - }, - "syft-json: returns error when input is not json": { - data: []byte(`foobar`), - format: FormatSyftJSON, - wantErr: true, - }, - } - for n, tc := range testCases { - t.Run(n, func(t *testing.T) { - gotPkgs, err := PackagesFromBOM(bytes.NewReader(tc.data), tc.format) - if err != nil && !tc.wantErr { - t.Fatalf("unexpected error: %s", err) - } - if err == nil && tc.wantErr { - t.Fatalf("expected error but got nil") - } - - if diff := cmp.Diff(gotPkgs, tc.wantPkgs); diff != "" { - t.Fatalf("unexpected packages:\n%s", diff) - } - }) - } -} diff --git a/internal/bom/cyclonedx.go b/internal/bom/cyclonedx.go new file mode 100644 index 0000000..753309b --- /dev/null +++ b/internal/bom/cyclonedx.go @@ -0,0 +1,163 @@ +package bom + +import ( + "errors" + "fmt" + "io" + + "github.com/CycloneDX/cyclonedx-go" + github_url "github.com/jetstack/tally/internal/github-url" + "github.com/jetstack/tally/internal/types" + "github.com/package-url/packageurl-go" +) + +type cdxBOM struct { + bom *cyclonedx.BOM +} + +func parseCycloneDX(r io.Reader, format cyclonedx.BOMFileFormat) (BOM, error) { + bom := &cyclonedx.BOM{} + if err := cyclonedx.NewBOMDecoder(r, format).Decode(bom); err != nil { + return nil, fmt.Errorf("decoding cyclonedx BOM: %w", err) + } + + return &cdxBOM{ + bom: bom, + }, nil +} + +// Packages returns all the supported packages in the BOM +func (b *cdxBOM) Packages() ([]types.Package, error) { + var pkgs []types.Package + if err := foreachComponentIn( + b.bom, + func(component cyclonedx.Component) error { + pkg, err := packageFromCycloneDXComponent(component) + if err != nil { + return err + } + if pkg == nil { + return nil + } + if containsPackage(pkgs, *pkg) { + return nil + } + pkgs = append(pkgs, *pkg) + + return nil + + }, + ); err != nil { + return nil, fmt.Errorf("finding packages in BOM: %w", err) + } + + return pkgs, nil +} + +// Repositories returns any repositories specified in the BOM for the provided +// package +func (b *cdxBOM) Repositories(pkg types.Package) ([]string, error) { + var repos []string + if err := foreachComponentIn( + b.bom, + func(component cyclonedx.Component) error { + p, err := packageFromCycloneDXComponent(component) + if err != nil { + return err + } + if p == nil { + return nil + } + if !p.Equals(pkg) { + return nil + } + if component.ExternalReferences != nil { + for _, ref := range *component.ExternalReferences { + switch ref.Type { + case cyclonedx.ERTypeVCS, cyclonedx.ERTypeDistribution, cyclonedx.ERTypeWebsite: + repo, err := github_url.ToRepository(ref.URL) + if err != nil { + continue + } + if !contains(repos, repo) { + repos = append(repos, repo) + } + } + } + } + + return nil + + }, + ); err != nil { + return nil, fmt.Errorf("finding repositories in BOM: %w", err) + } + + return repos, nil +} + +func foreachComponentIn(bom *cyclonedx.BOM, fn func(component cyclonedx.Component) error) error { + var components []cyclonedx.Component + if bom.Metadata != nil && bom.Metadata.Component != nil { + components = append(components, *bom.Metadata.Component) + } + if bom.Components != nil { + components = append(components, *bom.Components...) + } + return walkCycloneDXComponents(components, fn) +} + +func walkCycloneDXComponents(components []cyclonedx.Component, fn func(cyclonedx.Component) error) error { + for _, component := range components { + if err := fn(component); err != nil { + return err + } + if component.Components == nil { + continue + } + if err := walkCycloneDXComponents(*component.Components, fn); err != nil { + return err + } + } + + return nil +} + +func packageFromCycloneDXComponent(component cyclonedx.Component) (*types.Package, error) { + if component.PackageURL == "" { + return nil, nil + } + purl, err := packageurl.FromString(component.PackageURL) + if err != nil { + return nil, err + } + pkg, err := packageFromPurl(purl) + if errors.Is(err, ErrUnsupportedPackageType) { + return nil, nil + } + if err != nil { + return nil, err + } + + return pkg, nil +} + +func contains(vals []string, val string) bool { + for _, v := range vals { + if v == val { + return true + } + } + + return false +} + +func containsPackage(pkgs []types.Package, pkg types.Package) bool { + for _, p := range pkgs { + if p.Equals(pkg) { + return true + } + } + + return false +} diff --git a/internal/bom/cyclonedx_test.go b/internal/bom/cyclonedx_test.go new file mode 100644 index 0000000..447487d --- /dev/null +++ b/internal/bom/cyclonedx_test.go @@ -0,0 +1,572 @@ +package bom + +import ( + "encoding/xml" + "os" + "testing" + + "github.com/CycloneDX/cyclonedx-go" + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/types" +) + +func TestCycloneDXParse(t *testing.T) { + testCases := map[string]struct { + path string + format Format + wantBOM *cyclonedx.BOM + wantErr bool + }{ + "json is parsed successfully": { + path: "testdata/cdx.json", + format: FormatCycloneDXJSON, + wantBOM: &cyclonedx.BOM{ + BOMFormat: "CycloneDX", + SpecVersion: cyclonedx.SpecVersion1_4, + SerialNumber: "urn:uuid:5e0841b1-88e1-4dd8-b706-77457fb3e779", + Version: 1, + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + BOMRef: "1234567", + Type: "application", + Name: "foo/bar", + Version: "v0.2.5", + PackageURL: "pkg:golang/foo/bar@v0.2.5", + }, + }, + Components: &[]cyclonedx.Component{ + { + BOMRef: "0", + Type: "library", + Name: "HdrHistogram", + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + BOMRef: "1", + Type: "library", + Name: "adduser", + PackageURL: "pkg:deb/debian/adduser@3.118?arch=all&distro=debian-11", + }, + }, + }, + }, + "error is returned when parsing invalid json": { + path: "testdata/cdx.json.invalid", + format: FormatCycloneDXJSON, + wantErr: true, + }, + "xml": { + path: "testdata/cdx.xml", + format: FormatCycloneDXXML, + wantBOM: &cyclonedx.BOM{ + SpecVersion: cyclonedx.SpecVersion1_3, + SerialNumber: "urn:uuid:5e0841b1-88e1-4dd8-b706-77457fb3e779", + Version: 1, + XMLName: xml.Name{ + Space: "http://cyclonedx.org/schema/bom/1.3", + Local: "bom", + }, + XMLNS: "http://cyclonedx.org/schema/bom/1.3", + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + BOMRef: "1234567", + Type: "application", + Name: "foo/bar", + Version: "v0.2.5", + PackageURL: "pkg:golang/foo/bar@v0.2.5", + }, + }, + Components: &[]cyclonedx.Component{ + { + BOMRef: "0", + Type: "library", + Name: "HdrHistogram", + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + BOMRef: "1", + Type: "library", + Name: "adduser", + PackageURL: "pkg:deb/debian/adduser@3.118?arch=all&distro=debian-11", + }, + }, + }, + }, + "error is returned when parsing invalid xml": { + path: "testdata/cdx.xml.invalid", + format: FormatCycloneDXXML, + wantErr: true, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + r, err := os.Open(tc.path) + if err != nil { + t.Fatalf("unexpected error opening file: %s", err) + } + defer r.Close() + + bom, err := ParseBOM(r, tc.format) + if err != nil && !tc.wantErr { + t.Fatalf("unexpected error parsing BOM: %s", err) + } + if err == nil && tc.wantErr { + t.Fatalf("expected parsing BOM but got nil") + } + + if tc.wantErr { + return + } + + gotBOM := bom.(*cdxBOM).bom + if diff := cmp.Diff(tc.wantBOM, gotBOM); diff != "" { + t.Errorf("unexpected BOM:\n%s", diff) + } + }) + } +} + +func TestCycloneDXBOMPackages(t *testing.T) { + testCases := map[string]struct { + bom *cyclonedx.BOM + wantPackages []types.Package + }{ + "an error should not be produced for an empty BOM": { + bom: &cyclonedx.BOM{}, + }, + "an error should not be produced when metadata.component is nil": { + bom: &cyclonedx.BOM{ + Metadata: &cyclonedx.Metadata{}, + }, + }, + "packages should be discovered in metadata.component": { + bom: &cyclonedx.BOM{ + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + PackageURL: "pkg:golang/foo/bar@v0.2.5", + }, + }, + }, + wantPackages: []types.Package{ + { + System: "GO", + Name: "foo/bar", + }, + }, + }, + "packages should be discovered in metadata.component AND components": { + bom: &cyclonedx.BOM{ + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + PackageURL: "pkg:golang/foo/bar@v0.2.5", + }}, + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + PackageURL: "pkg:deb/debian/adduser@3.118?arch=all\u0026distro=debian-11", + }, + }, + }, + wantPackages: []types.Package{ + { + System: "GO", + Name: "foo/bar", + }, + { + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + }, + }, + "packages should be discovered in nested components in metadata.component AND components": { + bom: &cyclonedx.BOM{ + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + PackageURL: "pkg:golang/foo/bar@v0.2.5", + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/sigs.k8s.io/release-utils@v0.7.3", + }, + }, + }}, + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + Components: &[]cyclonedx.Component{ + { + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:maven/com.github.package-url/packageurl-java@1.4.1", + }, + }, + }, + }, + }, + }, + }, + wantPackages: []types.Package{ + { + System: "GO", + Name: "foo/bar", + }, + { + System: "GO", + Name: "sigs.k8s.io/release-utils", + }, + { + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + { + System: "MAVEN", + Name: "com.github.package-url:packageurl-java", + }, + }, + }, + "unsupported packages should be ignored": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:nuget/EnterpriseLibrary.Common@6.0.1304", + }, + }, + }, + }, + "components without a PackageURL should be ignored": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + Name: "foo/bar", + }, + }, + }, + }, + "duplicate packages should be ignored": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.8", + }, + { + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + }, + }, + wantPackages: []types.Package{ + { + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + }, + }, + "all supported types should be discovered": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + PackageURL: "pkg:golang/sigs.k8s.io/release-utils@v0.7.3", + }, + { + PackageURL: "pkg:npm/zwitch@2.0.2", + }, + { + PackageURL: "pkg:cargo/getrandom@0.2.7", + }, + { + PackageURL: "pkg:pypi/zope.interface@5.4.0", + }, + }, + }, + wantPackages: []types.Package{ + { + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + { + System: "GO", + Name: "sigs.k8s.io/release-utils", + }, + { + System: "NPM", + Name: "zwitch", + }, + { + System: "CARGO", + Name: "getrandom", + }, + { + System: "PYPI", + Name: "zope.interface", + }, + }, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + bom := &cdxBOM{ + bom: tc.bom, + } + gotPackages, err := bom.Packages() + if err != nil { + t.Fatalf("unexpected error getting packages from bom: %s", err) + } + if diff := cmp.Diff(tc.wantPackages, gotPackages); diff != "" { + t.Errorf("unexpected packages:\n%s", diff) + } + }) + } +} + +func TestCycloneDXBOMRepositories(t *testing.T) { + testCases := map[string]struct { + bom *cyclonedx.BOM + pkg types.Package + wantRepos []string + }{ + "repositories can be extracted from metadata.components": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + }, + }, + "repositories can be extracted from components": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + }, + }, + "repositories can be extracted from nested components": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/bar/foo@v0.1.1", + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + }, + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + }, + }, + "multiple repositories can be extracted from the same component": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + { + Type: cyclonedx.ERTypeVCS, + URL: "git@github.com:baz/bar", + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + "github.com/baz/bar", + }, + }, + "multiple repositories can be extracted from different components": { + bom: &cyclonedx.BOM{ + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + }, + }, + }, + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "git@github.com:baz/bar", + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + "github.com/baz/bar", + }, + }, + "repositories are deduplicated": { + bom: &cyclonedx.BOM{ + Metadata: &cyclonedx.Metadata{ + Component: &cyclonedx.Component{ + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + }, + }, + }, + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + }, + }, + "repositories are extracted and deduplicated from all supported types": { + bom: &cyclonedx.BOM{ + Components: &[]cyclonedx.Component{ + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeVCS, + URL: "https://github.com/bar/foo", + }, + { + Type: cyclonedx.ERTypeDistribution, + URL: "https://github.com/bar/foo.git", + }, + { + Type: cyclonedx.ERTypeWebsite, + URL: "http://github.com/bar/foo.git", + }, + }, + }, + { + PackageURL: "pkg:golang/foo/bar@v0.2.2", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeWebsite, + URL: "https://github.com/bar/foo.git", + }, + }, + }, + { + PackageURL: "pkg:golang/foo/bar@v0.2.2", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeDistribution, + URL: "https://github.com/foo/baz.git", + }, + }, + }, + { + PackageURL: "pkg:golang/foo/bar@v0.1.1", + ExternalReferences: &[]cyclonedx.ExternalReference{ + { + Type: cyclonedx.ERTypeWebsite, + URL: "https://github.com/foo/bar", + }, + }, + }, + }, + }, + pkg: types.Package{ + System: "GO", + Name: "foo/bar", + }, + wantRepos: []string{ + "github.com/bar/foo", + "github.com/foo/baz", + "github.com/foo/bar", + }, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + bom := &cdxBOM{ + bom: tc.bom, + } + gotRepos, err := bom.Repositories(tc.pkg) + if err != nil { + t.Fatalf("unexpected error getting repositories from bom: %s", err) + } + if diff := cmp.Diff(tc.wantRepos, gotRepos); diff != "" { + t.Errorf("unexpected packages:\n%s", diff) + } + }) + } +} diff --git a/internal/bom/purl.go b/internal/bom/purl.go new file mode 100644 index 0000000..75240c3 --- /dev/null +++ b/internal/bom/purl.go @@ -0,0 +1,55 @@ +package bom + +import ( + "errors" + "strings" + + "github.com/jetstack/tally/internal/types" + "github.com/package-url/packageurl-go" +) + +var ( + // ErrUnsupportedPackageType is returned when parsing a package of an + // unsupported type + ErrUnsupportedPackageType = errors.New("unsupported package type") +) + +func packageFromPurl(purl packageurl.PackageURL) (*types.Package, error) { + switch purl.Type { + case packageurl.TypeNPM: + name := purl.Name + if purl.Namespace != "" { + name = purl.Namespace + "/" + name + } + return &types.Package{ + System: "NPM", + Name: name, + }, nil + case packageurl.TypeGolang: + name := purl.Name + if purl.Namespace != "" { + name = purl.Namespace + "/" + purl.Name + } + return &types.Package{ + System: "GO", + Name: name, + }, nil + case packageurl.TypeMaven: + return &types.Package{ + System: "MAVEN", + Name: strings.Join([]string{purl.Namespace, purl.Name}, ":"), + }, nil + case packageurl.TypePyPi: + return &types.Package{ + System: "PYPI", + Name: purl.Name, + }, nil + case "cargo": + return &types.Package{ + System: "CARGO", + Name: purl.Name, + }, nil + default: + return nil, ErrUnsupportedPackageType + } +} diff --git a/internal/bom/purl_test.go b/internal/bom/purl_test.go new file mode 100644 index 0000000..07aff46 --- /dev/null +++ b/internal/bom/purl_test.go @@ -0,0 +1,74 @@ +package bom + +import ( + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/types" + "github.com/package-url/packageurl-go" +) + +func TestPackageFromPurl(t *testing.T) { + testCases := []struct { + purl string + wantPkg *types.Package + wantErr error + }{ + { + purl: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + wantPkg: &types.Package{ + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + }, + { + purl: "pkg:golang/sigs.k8s.io/release-utils@v0.7.3", + wantPkg: &types.Package{ + System: "GO", + Name: "sigs.k8s.io/release-utils", + }, + }, + { + purl: "pkg:npm/zwitch@2.0.2", + wantPkg: &types.Package{ + System: "NPM", + Name: "zwitch", + }, + }, + { + purl: "pkg:cargo/getrandom@0.2.7", + wantPkg: &types.Package{ + System: "CARGO", + Name: "getrandom", + }, + }, + { + purl: "pkg:pypi/zope.interface@5.4.0", + wantPkg: &types.Package{ + System: "PYPI", + Name: "zope.interface", + }, + }, + { + purl: "pkg:nuget/EnterpriseLibrary.Common@6.0.1304", + wantErr: ErrUnsupportedPackageType, + }, + } + for _, tc := range testCases { + purl, err := packageurl.FromString(tc.purl) + if err != nil { + t.Fatalf("unexpected error parsing purl: %s", err) + } + + gotPkg, err := packageFromPurl(purl) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("unexpected error; wanted %s but got %s", tc.wantErr, err) + } + + if diff := cmp.Diff(tc.wantPkg, gotPkg); diff != "" { + t.Errorf("unexpected package:\n%s", diff) + } + + } +} diff --git a/internal/bom/syft.go b/internal/bom/syft.go new file mode 100644 index 0000000..9b18cc7 --- /dev/null +++ b/internal/bom/syft.go @@ -0,0 +1,67 @@ +package bom + +import ( + "encoding/json" + "errors" + "io" + + "github.com/jetstack/tally/internal/types" + "github.com/package-url/packageurl-go" +) + +type syftJSON struct { + Artifacts []syftArtifact `json:"artifacts"` +} + +type syftArtifact struct { + Purl string `json:"purl"` +} + +type syftBOM struct { + bom *syftJSON +} + +func parseSyftJSON(r io.Reader) (BOM, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + doc := &syftJSON{} + if err := json.Unmarshal(data, doc); err != nil { + return nil, err + } + return &syftBOM{ + bom: doc, + }, nil +} + +// Packages returns all the supported packages in the BOM +func (bom *syftBOM) Packages() ([]types.Package, error) { + var pkgs []types.Package + for _, a := range bom.bom.Artifacts { + if a.Purl == "" { + continue + } + purl, err := packageurl.FromString(a.Purl) + if err != nil { + return nil, err + } + pkg, err := packageFromPurl(purl) + if errors.Is(err, ErrUnsupportedPackageType) { + continue + } + if err != nil { + return nil, err + } + if containsPackage(pkgs, *pkg) { + continue + } + pkgs = append(pkgs, *pkg) + } + return pkgs, nil +} + +// Repositories doesn't return anything for the syft-json format +func (bom *syftBOM) Repositories(pkg types.Package) ([]string, error) { + return []string{}, nil +} diff --git a/internal/bom/syft_test.go b/internal/bom/syft_test.go new file mode 100644 index 0000000..afbf533 --- /dev/null +++ b/internal/bom/syft_test.go @@ -0,0 +1,169 @@ +package bom + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/types" +) + +func TestSyftParseBOM(t *testing.T) { + testCases := map[string]struct { + path string + format Format + wantBOM *syftJSON + wantErr bool + }{ + "json is parsed successfully": { + path: "testdata/syft.json", + format: FormatSyftJSON, + wantBOM: &syftJSON{ + Artifacts: []syftArtifact{ + { + Purl: "pkg:golang/foo/bar@v0.2.5", + }, + { + Purl: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + }, + }, + }, + "error is returned when parsing invalid json": { + path: "testdata/syft.json.invalid", + format: FormatSyftJSON, + wantErr: true, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + r, err := os.Open(tc.path) + if err != nil { + t.Fatalf("unexpected error opening file: %s", err) + } + defer r.Close() + + bom, err := ParseBOM(r, tc.format) + if err != nil && !tc.wantErr { + t.Fatalf("unexpected error parsing BOM: %s", err) + } + if err == nil && tc.wantErr { + t.Fatalf("expected parsing BOM but got nil") + } + + if tc.wantErr { + return + } + + gotBOM := bom.(*syftBOM).bom + if diff := cmp.Diff(tc.wantBOM, gotBOM); diff != "" { + t.Errorf("unexpected BOM:\n%s", diff) + } + }) + } +} + +func TestSyftBOMPackages(t *testing.T) { + testCases := map[string]struct { + bom *syftJSON + wantPackages []types.Package + }{ + "an error should not be produced for an empty BOM": { + bom: &syftJSON{}, + }, + "unsupported packages should be ignored": { + bom: &syftJSON{ + Artifacts: []syftArtifact{ + { + Purl: "pkg:nuget/EnterpriseLibrary.Common@6.0.1304", + }, + }, + }, + }, + "components without a Purl should be ignored": { + bom: &syftJSON{ + Artifacts: []syftArtifact{ + {}, + }, + }, + }, + "duplicate packages should be ignored": { + bom: &syftJSON{ + Artifacts: []syftArtifact{ + { + Purl: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.8", + }, + { + Purl: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + Purl: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + }, + }, + wantPackages: []types.Package{ + { + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + }, + }, + "all supported types should be discovered": { + bom: &syftJSON{ + Artifacts: []syftArtifact{ + { + Purl: "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9", + }, + { + Purl: "pkg:golang/sigs.k8s.io/release-utils@v0.7.3", + }, + { + Purl: "pkg:npm/zwitch@2.0.2", + }, + { + Purl: "pkg:cargo/getrandom@0.2.7", + }, + { + Purl: "pkg:pypi/zope.interface@5.4.0", + }, + }, + }, + wantPackages: []types.Package{ + { + System: "MAVEN", + Name: "org.hdrhistogram:HdrHistogram", + }, + { + System: "GO", + Name: "sigs.k8s.io/release-utils", + }, + { + System: "NPM", + Name: "zwitch", + }, + { + System: "CARGO", + Name: "getrandom", + }, + { + System: "PYPI", + Name: "zope.interface", + }, + }, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + bom := &syftBOM{ + bom: tc.bom, + } + gotPackages, err := bom.Packages() + if err != nil { + t.Fatalf("unexpected error getting packages from bom: %s", err) + } + if diff := cmp.Diff(tc.wantPackages, gotPackages); diff != "" { + t.Errorf("unexpected packages:\n%s", diff) + } + }) + } +} diff --git a/internal/bom/testdata/cdx.json b/internal/bom/testdata/cdx.json new file mode 100644 index 0000000..d368b1b --- /dev/null +++ b/internal/bom/testdata/cdx.json @@ -0,0 +1,29 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "serialNumber": "urn:uuid:5e0841b1-88e1-4dd8-b706-77457fb3e779", + "version": 1, + "metadata": { + "component": { + "bom-ref": "1234567", + "type": "application", + "name": "foo/bar", + "version": "v0.2.5", + "purl": "pkg:golang/foo/bar@v0.2.5" + } + }, + "components": [ + { + "bom-ref": "0", + "type": "library", + "name": "HdrHistogram", + "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9" + }, + { + "bom-ref": "1", + "type": "library", + "name": "adduser", + "purl": "pkg:deb/debian/adduser@3.118?arch=all\u0026distro=debian-11" + } + ] +} diff --git a/internal/bom/testdata/cdx.json.invalid b/internal/bom/testdata/cdx.json.invalid new file mode 100644 index 0000000..494a673 --- /dev/null +++ b/internal/bom/testdata/cdx.json.invalid @@ -0,0 +1,29 @@ +{ + "bomFormat": "CycloneDX" + "specVersion": "1.4", + "serialNumber": "urn:uuid:5e0841b1-88e1-4dd8-b706-77457fb3e779", + "version": 1, + "metadata": { + "component": { + "bom-ref": "1234567", + "type": "application", + "name": "foo/bar", + "version": "v0.2.5", + "purl": "pkg:golang/foo/bar@v0.2.5" + } + }, + "components": [ + { + "bom-ref": "0", + "type": "library", + "name": "HdrHistogram", + "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9" + }, + { + "bom-ref": "1", + "type": "library", + "name": "adduser", + "purl": "pkg:deb/debian/adduser@3.118?arch=all\u0026distro=debian-11" + } + ] +} diff --git a/internal/bom/testdata/cdx.xml b/internal/bom/testdata/cdx.xml new file mode 100644 index 0000000..eccfb8e --- /dev/null +++ b/internal/bom/testdata/cdx.xml @@ -0,0 +1,20 @@ + + + + + foo/bar + v0.2.5 + pkg:golang/foo/bar@v0.2.5 + + + + + HdrHistogram + pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9 + + + adduser + pkg:deb/debian/adduser@3.118?arch=all&distro=debian-11 + + + diff --git a/internal/bom/testdata/cdx.xml.invalid b/internal/bom/testdata/cdx.xml.invalid new file mode 100644 index 0000000..7c4575f --- /dev/null +++ b/internal/bom/testdata/cdx.xml.invalid @@ -0,0 +1,19 @@ + + + + + foo/bar + v0.2.5 + pkg:golang/foo/bar@v0.2.5 + + + + HdrHistogram + pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9 + + + adduser + pkg:deb/debian/adduser@3.118?arch=all&distro=debian-11 + + + diff --git a/internal/bom/testdata/syft.json b/internal/bom/testdata/syft.json new file mode 100644 index 0000000..221c705 --- /dev/null +++ b/internal/bom/testdata/syft.json @@ -0,0 +1,14 @@ +{ + "artifacts": [ + { + "id": "0", + "name": "foo", + "purl": "pkg:golang/foo/bar@v0.2.5" + }, + { + "id": "1", + "name": "HdrHistogram", + "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9" + } + ] +} diff --git a/internal/bom/testdata/syft.json.invalid b/internal/bom/testdata/syft.json.invalid new file mode 100644 index 0000000..c2a8777 --- /dev/null +++ b/internal/bom/testdata/syft.json.invalid @@ -0,0 +1,14 @@ +{ + "artifacts": [ + { + "id": "0", + "name": "foo" + "purl": "pkg:golang/foo/bar@v0.2.5" + }, + { + "id": "1", + "name": "HdrHistogram", + "purl": "pkg:maven/org.hdrhistogram/HdrHistogram@2.1.9" + } + ] +} diff --git a/internal/db/db.go b/internal/db/db.go index 0f3d51d..6cd09df 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -22,15 +22,12 @@ type DB interface { // DBReader reads from the database type DBReader interface { + RepositoryReader + // GetChecks retrieves check scores for a repository. Returns // ErrNotFound if no checks are found. GetChecks(context.Context, string) ([]Check, error) - // GetRepositories returns any repositories associated with the package - // indicated by system and name. Returns ErrNotFound if there are no matching - // repositories. - GetRepositories(context.Context, string, string) ([]string, error) - // GetScore retrieves scorecard scores for a list of repositories GetScores(context.Context, ...string) ([]Score, error) } @@ -52,6 +49,11 @@ type DBWriter interface { AddScores(context.Context, ...Score) error } +// RepositoryReader reads repositories from the database +type RepositoryReader interface { + GetRepositories(context.Context, string, string) ([]string, error) +} + // Package is a package associated with a repository type Package struct { System string diff --git a/internal/github-url/repository.go b/internal/github-url/repository.go new file mode 100644 index 0000000..2e148f5 --- /dev/null +++ b/internal/github-url/repository.go @@ -0,0 +1,22 @@ +package github_url + +import ( + "fmt" + "regexp" + "strings" +) + +var ( + ghRegex = regexp.MustCompile(`(?:https|git)(?:://|@)github\.com[/:]([^/:#]+)/([^/#]*).*`) + ghSuffixRegex = regexp.MustCompile(`(\.git/?)?(\.git|\?.*|#.*)?$`) +) + +// ToRepository parses a github url from a number of different formats into our +// expected repository format: github.com//. +func ToRepository(u string) (string, error) { + matches := ghRegex.FindStringSubmatch(ghSuffixRegex.ReplaceAllString(u, "")) + if len(matches) < 3 { + return "", fmt.Errorf("couldn't parse url") + } + return strings.Join([]string{"github.com", matches[1], matches[2]}, "/"), nil +} diff --git a/internal/github-url/repository_test.go b/internal/github-url/repository_test.go new file mode 100644 index 0000000..6ea2270 --- /dev/null +++ b/internal/github-url/repository_test.go @@ -0,0 +1,138 @@ +package github_url + +import "testing" + +func TestToRepository(t *testing.T) { + testCases := []struct { + url string + wantRepo string + wantErr bool + }{ + { + url: "https://github.com/foo/bar", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar/tree/main/baz", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar#baz", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar/", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar.git", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar.git/", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar.git?ref=baz", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar.git?ref=baz", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar?ref=something", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo/bar#something", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar/", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar.git", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar.git/", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar.git?ref=baz", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar?ref=something", + wantRepo: "github.com/foo/bar", + }, + { + url: "git://github.com/foo/bar#something", + wantRepo: "github.com/foo/bar", + }, + { + url: "git@github.com:foo/bar.git", + wantRepo: "github.com/foo/bar", + }, + { + url: "git@github.com:foo/bar.git", + wantRepo: "github.com/foo/bar", + }, + { + url: "git@github.com:foo/bar.git/", + wantRepo: "github.com/foo/bar", + }, + { + url: "git@github.com:foo/bar.git?ref=something", + wantRepo: "github.com/foo/bar", + }, + { + url: "git@github.com:foo/bar.git#something", + wantRepo: "github.com/foo/bar", + }, + { + url: "https://github.com/foo", + wantErr: true, + }, + { + url: "https://gitlab.com/foo/bar", + wantErr: true, + }, + + { + url: "git://gitlab.com/foo/bar.git", + wantErr: true, + }, + { + url: "git@gitlab.com:foo/bar.git", + wantErr: true, + }, + { + url: "github.com", + wantErr: true, + }, + } + for _, tc := range testCases { + gotRepo, err := ToRepository(tc.url) + if err != nil && !tc.wantErr { + t.Fatalf("unexpected error parsing %q: %s", tc.url, err) + } + if err == nil && tc.wantErr { + t.Fatalf("expected error but got nil") + } + + if gotRepo != tc.wantRepo { + t.Fatalf("unexpected repo parsing %q; got %q but wanted %q", tc.url, gotRepo, tc.wantRepo) + } + } +} diff --git a/internal/repositories/bom.go b/internal/repositories/bom.go new file mode 100644 index 0000000..35d90f6 --- /dev/null +++ b/internal/repositories/bom.go @@ -0,0 +1,24 @@ +package repositories + +import ( + "context" + + "github.com/jetstack/tally/internal/bom" + "github.com/jetstack/tally/internal/types" +) + +type bomMapper struct { + bom bom.BOM +} + +// BOMMapper returns a mapper that extracts repositories from a BOM +func BOMMapper(bom bom.BOM) Mapper { + return &bomMapper{ + bom: bom, + } +} + +// Repositories returns any repositories for components in a BOM +func (m *bomMapper) Repositories(ctx context.Context, pkg types.Package) ([]string, error) { + return m.bom.Repositories(pkg) +} diff --git a/internal/repositories/bom_test.go b/internal/repositories/bom_test.go new file mode 100644 index 0000000..65c04b5 --- /dev/null +++ b/internal/repositories/bom_test.go @@ -0,0 +1,69 @@ +package repositories + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/types" +) + +type mockBOM struct { + repositories func(pkg types.Package) ([]string, error) +} + +func (m *mockBOM) Packages() ([]types.Package, error) { + return nil, nil +} + +func (m *mockBOM) Repositories(pkg types.Package) ([]string, error) { + return m.repositories(pkg) +} + +func TestBOMMapperRepositories(t *testing.T) { + type testCase struct { + repoFn func(types.Package) ([]string, error) + wantRepos []string + wantErr error + } + testCases := map[string]func(t *testing.T) *testCase{ + "should return repositories retrieved from BOM": func(t *testing.T) *testCase { + wantRepos := []string{"a", "b"} + return &testCase{ + repoFn: func(types.Package) ([]string, error) { + return wantRepos, nil + }, + wantRepos: wantRepos, + } + }, + "should return error from BOM": func(t *testing.T) *testCase { + wantErr := errors.New("repositories error") + return &testCase{ + repoFn: func(types.Package) ([]string, error) { + return []string{}, wantErr + }, + wantErr: wantErr, + } + }, + } + for n, setup := range testCases { + t.Run(n, func(t *testing.T) { + tc := setup(t) + bom := &mockBOM{tc.repoFn} + gotRepos, err := BOMMapper(bom).Repositories(context.Background(), types.Package{ + System: "GO", + Name: "foo/bar", + }) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("unexpected error: %s", err) + } + if tc.wantErr != nil { + return + } + if diff := cmp.Diff(tc.wantRepos, gotRepos); diff != "" { + t.Errorf("unexpected repositories:\n%s", diff) + } + }) + } +} diff --git a/internal/repositories/db.go b/internal/repositories/db.go new file mode 100644 index 0000000..a578c06 --- /dev/null +++ b/internal/repositories/db.go @@ -0,0 +1,34 @@ +package repositories + +import ( + "context" + "errors" + "fmt" + + "github.com/jetstack/tally/internal/db" + "github.com/jetstack/tally/internal/types" +) + +type dbMapper struct { + d db.RepositoryReader +} + +// DBMapper returns a mapper that retrieves repositories from the tally database +func DBMapper(d db.RepositoryReader) Mapper { + return &dbMapper{ + d: d, + } +} + +// Repositories gets repositories from the tally database +func (m *dbMapper) Repositories(ctx context.Context, pkg types.Package) ([]string, error) { + repos, err := m.d.GetRepositories(ctx, pkg.System, pkg.Name) + if errors.Is(err, db.ErrNotFound) { + return []string{}, nil + } + if err != nil { + return []string{}, fmt.Errorf("database mapper: %w", err) + } + + return repos, nil +} diff --git a/internal/repositories/db_test.go b/internal/repositories/db_test.go new file mode 100644 index 0000000..024b5af --- /dev/null +++ b/internal/repositories/db_test.go @@ -0,0 +1,73 @@ +package repositories + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/db" + "github.com/jetstack/tally/internal/types" +) + +type mockDB struct { + getRepositories func(ctx context.Context, system, name string) ([]string, error) +} + +func (d *mockDB) GetRepositories(ctx context.Context, system, name string) ([]string, error) { + return d.getRepositories(ctx, system, name) +} + +func TestDBMapperRepositories(t *testing.T) { + type testCase struct { + getRepos func(ctx context.Context, system, name string) ([]string, error) + wantRepos []string + wantErr error + } + testCases := map[string]func(t *testing.T) *testCase{ + "should return repos from GetRepositories": func(t *testing.T) *testCase { + return &testCase{ + getRepos: func(ctx context.Context, system, name string) ([]string, error) { + return []string{"a", "b", "c"}, nil + }, + wantRepos: []string{"a", "b", "c"}, + } + }, + "should return error from GetRepositories": func(t *testing.T) *testCase { + wantErr := errors.New("test error") + return &testCase{ + getRepos: func(ctx context.Context, system, name string) ([]string, error) { + return []string{}, wantErr + }, + wantErr: wantErr, + } + }, + "should ignore db.ErrNotFound": func(t *testing.T) *testCase { + return &testCase{ + getRepos: func(ctx context.Context, system, name string) ([]string, error) { + return []string{}, db.ErrNotFound + }, + wantRepos: []string{}, + } + }, + } + for n, setup := range testCases { + t.Run(n, func(t *testing.T) { + tc := setup(t) + + gotRepos, err := DBMapper(&mockDB{tc.getRepos}).Repositories(context.Background(), types.Package{ + System: "GO", + Name: "foo/bar", + }) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("unexpected error: %s", err) + } + if tc.wantErr != nil { + return + } + if diff := cmp.Diff(tc.wantRepos, gotRepos); diff != "" { + t.Errorf("unexpected repositories:\n%s", diff) + } + }) + } +} diff --git a/internal/repositories/mapper.go b/internal/repositories/mapper.go new file mode 100644 index 0000000..8b38e73 --- /dev/null +++ b/internal/repositories/mapper.go @@ -0,0 +1,56 @@ +package repositories + +import ( + "context" + "fmt" + + "github.com/jetstack/tally/internal/types" +) + +// Mapper maps a package to associated repositories +type Mapper interface { + // Repositories returns repositories for the given package. + Repositories(ctx context.Context, pkg types.Package) ([]string, error) +} + +type mapper struct { + mappers []Mapper +} + +// From returns a mapper that attempts to retrieve repositories from +// multiple mappers +func From(mappers ...Mapper) Mapper { + return &mapper{ + mappers: mappers, + } +} + +// GetRepositories iterates through the mappers and returns all the repositories +func (m *mapper) Repositories(ctx context.Context, pkg types.Package) ([]string, error) { + var repositories []string + for _, mpr := range m.mappers { + repos, err := mpr.Repositories(ctx, pkg) + if err != nil { + return []string{}, fmt.Errorf("getting repositories: %w", err) + } + for _, repo := range repos { + if contains(repositories, repo) { + continue + } + + repositories = append(repositories, repo) + } + } + + return repositories, nil +} + +func contains(vals []string, val string) bool { + for _, v := range vals { + if v == val { + return true + } + } + + return false +} diff --git a/internal/repositories/mapper_test.go b/internal/repositories/mapper_test.go new file mode 100644 index 0000000..dc3ec02 --- /dev/null +++ b/internal/repositories/mapper_test.go @@ -0,0 +1,156 @@ +package repositories + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/types" +) + +type mockMapper struct { + repositories func(ctx context.Context, pkg types.Package) ([]string, error) +} + +func (m *mockMapper) Repositories(ctx context.Context, pkg types.Package) ([]string, error) { + return m.repositories(ctx, pkg) +} + +func TestMultiMapperRepositories(t *testing.T) { + type testCase struct { + pkg types.Package + repoFns []func(ctx context.Context, pkg types.Package) ([]string, error) + wantRepos []string + wantErr error + } + testCases := map[string]func(t *testing.T) *testCase{ + "should iterate through all mappers, regardless of whether they return anything": func(t *testing.T) *testCase { + return &testCase{ + repoFns: []func(ctx context.Context, pkg types.Package) ([]string, error){ + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"a"}, nil + }, + }, + wantRepos: []string{"a"}, + } + }, + "should aggregate repositories from multiple mappers": func(t *testing.T) *testCase { + return &testCase{ + repoFns: []func(ctx context.Context, pkg types.Package) ([]string, error){ + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"a", "b"}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"c"}, nil + }, + }, + wantRepos: []string{"a", "b", "c"}, + } + }, + "should deduplicate repositories from different mappers": func(t *testing.T) *testCase { + return &testCase{ + repoFns: []func(ctx context.Context, pkg types.Package) ([]string, error){ + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"a", "b"}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"c", "b"}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"a"}, nil + }, + }, + wantRepos: []string{"a", "b", "c"}, + } + }, + "should deduplicate repositories from the same mapper": func(t *testing.T) *testCase { + return &testCase{ + repoFns: []func(ctx context.Context, pkg types.Package) ([]string, error){ + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"a", "b", "c", "c", "a"}, nil + }, + }, + wantRepos: []string{"a", "b", "c"}, + } + }, + "should pass the same package to each mapper": func(t *testing.T) *testCase { + var wantPkg *types.Package + return &testCase{ + repoFns: []func(ctx context.Context, pkg types.Package) ([]string, error){ + func(ctx context.Context, pkg types.Package) ([]string, error) { + wantPkg = &pkg + return []string{}, nil + }, + func(ctx context.Context, gotPkg types.Package) ([]string, error) { + if !wantPkg.Equals(gotPkg) { + t.Fatalf("unexpected package in second mapper; want %v got %v", wantPkg, gotPkg) + } + return []string{}, nil + }, + func(ctx context.Context, gotPkg types.Package) ([]string, error) { + if !wantPkg.Equals(gotPkg) { + t.Fatalf("unexpected package in third mapper; want %v got %v", wantPkg, gotPkg) + } + return []string{"a"}, nil + }, + }, + wantRepos: []string{"a"}, + } + }, + "should return an error from any of the mappers": func(t *testing.T) *testCase { + wantErr := errors.New("mapper error") + return &testCase{ + repoFns: []func(ctx context.Context, pkg types.Package) ([]string, error){ + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{"a", "b"}, nil + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + return []string{}, wantErr + }, + func(ctx context.Context, pkg types.Package) ([]string, error) { + t.Errorf("unexpected call to third mapper") + return []string{"a"}, nil + }, + }, + wantRepos: []string{"a", "b", "c"}, + wantErr: wantErr, + } + }, + } + for n, setup := range testCases { + t.Run(n, func(t *testing.T) { + tc := setup(t) + + var mappers []Mapper + for _, fn := range tc.repoFns { + mappers = append(mappers, &mockMapper{fn}) + } + + gotRepos, err := From(mappers...).Repositories(context.Background(), types.Package{ + System: "GO", + Name: "foo/bar", + }) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("unexpected error: %s", err) + } + + if tc.wantErr != nil { + return + } + + if diff := cmp.Diff(tc.wantRepos, gotRepos); diff != "" { + t.Errorf("unexpected repositories:\n%s", diff) + } + }) + } +} diff --git a/internal/repositories/pkg.go b/internal/repositories/pkg.go new file mode 100644 index 0000000..31d7c3e --- /dev/null +++ b/internal/repositories/pkg.go @@ -0,0 +1,33 @@ +package repositories + +import ( + "context" + "strings" + + "github.com/jetstack/tally/internal/types" +) + +// PackageMapper is a mapper that infers repositories from information that is +// apparent from the package, i.e the name +var PackageMapper Mapper = &pkgMapper{} + +type pkgMapper struct{} + +// Repositories tries to infer repositories from information that is apparent +// from the package itself +func (m *pkgMapper) Repositories(ctx context.Context, pkg types.Package) ([]string, error) { + switch pkg.System { + case "GO": + if !strings.HasPrefix(pkg.Name, "github.com/") { + return []string{}, nil + } + parts := strings.Split(pkg.Name, "/") + if len(parts) < 3 { + return []string{}, nil + } + + return []string{strings.Join([]string{parts[0], parts[1], parts[2]}, "/")}, nil + default: + return []string{}, nil + } +} diff --git a/internal/repositories/pkg_test.go b/internal/repositories/pkg_test.go new file mode 100644 index 0000000..4d522af --- /dev/null +++ b/internal/repositories/pkg_test.go @@ -0,0 +1,49 @@ +package repositories + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/jetstack/tally/internal/types" +) + +func TestPackageMapperRepositories(t *testing.T) { + testCases := map[string]struct { + pkg types.Package + wantRepos []string + }{ + "should extract repository Go package hosted on github.com": { + pkg: types.Package{ + System: "GO", + Name: "github.com/foo/bar", + }, + wantRepos: []string{"github.com/foo/bar"}, + }, + "shouldn't extract repository from Go package hosted on gitlab.com": { + pkg: types.Package{ + System: "GO", + Name: "gitlab.com/foo/bar", + }, + wantRepos: []string{}, + }, + "shouldn't extract repository from non-Go package": { + pkg: types.Package{ + System: "NPM", + Name: "github.com/foo/bar", + }, + wantRepos: []string{}, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + gotRepos, err := PackageMapper.Repositories(context.Background(), tc.pkg) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if diff := cmp.Diff(tc.wantRepos, gotRepos); diff != "" { + t.Errorf("unexpected repos:\n%s", diff) + } + }) + } +}