From 5cb180c5ae33f1fc5bf815a909eb115795f9c09b Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Thu, 24 Oct 2024 00:17:54 +0530
Subject: [PATCH 1/2] Migrate pysec importer to aboutcode pipeline

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 vulnerabilities/importers/__init__.py         |  4 +-
 vulnerabilities/importers/pysec.py            | 44 -------------
 vulnerabilities/pipelines/pypa_importer.py    |  2 +-
 vulnerabilities/pipelines/pysec_importer.py   | 65 +++++++++++++++++++
 .../test_pysec_importer_pipeline.py}          | 17 +++--
 5 files changed, 76 insertions(+), 56 deletions(-)
 delete mode 100644 vulnerabilities/importers/pysec.py
 create mode 100644 vulnerabilities/pipelines/pysec_importer.py
 rename vulnerabilities/tests/{test_pysec.py => pipelines/test_pysec_importer_pipeline.py} (72%)

diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py
index 512e7a39c..3394dd989 100644
--- a/vulnerabilities/importers/__init__.py
+++ b/vulnerabilities/importers/__init__.py
@@ -26,7 +26,6 @@
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
 from vulnerabilities.importers import project_kb_msr2019
-from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
 from vulnerabilities.importers import ruby
@@ -42,9 +41,9 @@
 from vulnerabilities.pipelines import npm_importer
 from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
+from vulnerabilities.pipelines import pysec_importer
 
 IMPORTERS_REGISTRY = [
-    pysec.PyPIImporter,
     alpine_linux.AlpineImporter,
     openssl.OpensslImporter,
     redhat.RedhatImporter,
@@ -78,6 +77,7 @@
     gitlab_importer.GitLabImporterPipeline,
     github_importer.GitHubAPIImporterPipeline,
     nvd_importer.NVDImporterPipeline,
+    pysec_importer.PyPIImporterPipeline,
 ]
 
 IMPORTERS_REGISTRY = {
diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py
deleted file mode 100644
index 84a052f35..000000000
--- a/vulnerabilities/importers/pysec.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# VulnerableCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/aboutcode-org/vulnerablecode for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-import json
-import logging
-from io import BytesIO
-from typing import Iterable
-from zipfile import ZipFile
-
-import requests
-
-from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import Importer
-from vulnerabilities.importers.osv import parse_advisory_data
-
-logger = logging.getLogger(__name__)
-
-
-class PyPIImporter(Importer):
-    license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE"
-    spdx_license_expression = "CC-BY-4.0"
-    importer_name = "PyPI Importer"
-
-    def advisory_data(self) -> Iterable[AdvisoryData]:
-        """
-        Yield AdvisoryData using a zipped data dump of OSV data
-        """
-        url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
-        response = requests.get(url).content
-        with ZipFile(BytesIO(response)) as zip_file:
-            for file_name in zip_file.namelist():
-                if not file_name.startswith("PYSEC-"):
-                    logger.error(f"Unsupported PyPI advisory data file: {file_name}")
-                    continue
-                with zip_file.open(file_name) as f:
-                    vul_info = json.load(f)
-                    yield parse_advisory_data(
-                        raw_data=vul_info, supported_ecosystems=["pypi"], advisory_url=url
-                    )
diff --git a/vulnerabilities/pipelines/pypa_importer.py b/vulnerabilities/pipelines/pypa_importer.py
index 68d4615b9..bdda50c94 100644
--- a/vulnerabilities/pipelines/pypa_importer.py
+++ b/vulnerabilities/pipelines/pypa_importer.py
@@ -6,7 +6,7 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import logging
+
 from pathlib import Path
 from typing import Iterable
 
diff --git a/vulnerabilities/pipelines/pysec_importer.py b/vulnerabilities/pipelines/pysec_importer.py
new file mode 100644
index 000000000..c91e3be7b
--- /dev/null
+++ b/vulnerabilities/pipelines/pysec_importer.py
@@ -0,0 +1,65 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from io import BytesIO
+from typing import Iterable
+from zipfile import ZipFile
+
+import requests
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importers.osv import parse_advisory_data
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+
+
+class PyPIImporterPipeline(VulnerableCodeBaseImporterPipeline):
+    """Collect advisories from PyPI."""
+
+    pipeline_id = "pysec_importer"
+
+    license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE"
+    url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
+    spdx_license_expression = "CC-BY-4.0"
+    importer_name = "PyPI Importer"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.fetch_zip,
+            cls.collect_and_store_advisories,
+            cls.import_new_advisories,
+        )
+
+    def fetch_zip(self):
+        self.log(f"Fetching `{self.url}`")
+        self.advisory_zip = requests.get(self.url).content
+
+    def advisories_count(self) -> int:
+        with ZipFile(BytesIO(self.advisory_zip)) as zip:
+            advisory_count = sum(1 for file in zip.namelist() if file.startswith("PYSEC-"))
+        return advisory_count
+
+    def collect_advisories(self) -> Iterable[AdvisoryData]:
+        """Yield AdvisoryData using a zipped data dump of OSV data"""
+        with ZipFile(BytesIO(self.advisory_zip)) as zip_file:
+            for file_name in zip_file.namelist():
+                if not file_name.startswith("PYSEC-"):
+                    self.log(
+                        f"Unsupported PyPI advisory data file: {file_name}",
+                        level=logging.ERROR,
+                    )
+                    continue
+                with zip_file.open(file_name) as f:
+                    vul_info = json.load(f)
+                    yield parse_advisory_data(
+                        raw_data=vul_info,
+                        supported_ecosystems=["pypi"],
+                        advisory_url=self.url,
+                    )
diff --git a/vulnerabilities/tests/test_pysec.py b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py
similarity index 72%
rename from vulnerabilities/tests/test_pysec.py
rename to vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py
index d8e6e1c5b..b596c8f1d 100644
--- a/vulnerabilities/tests/test_pysec.py
+++ b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py
@@ -7,24 +7,23 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 import json
-import os
+from pathlib import Path
 from unittest import TestCase
 
 from vulnerabilities.importers.osv import parse_advisory_data
 from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN
 from vulnerabilities.tests.util_tests import check_results_against_json
 
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_DATA = os.path.join(BASE_DIR, "test_data/pysec")
+TEST_DATA = Path(__file__).parent.parent / "test_data" / "pysec"
 
 
 class TestPyPIImporter(TestCase):
     def test_to_advisories_with_summary(self):
-        with open(os.path.join(TEST_DATA, "pysec-advisories_with_summary.json")) as f:
+        with open(TEST_DATA / "pysec-advisories_with_summary.json") as f:
             mock_response = json.load(f)
         results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict()
 
-        expected_file = os.path.join(TEST_DATA, "pysec-advisories_with_summary-expected.json")
+        expected_file = TEST_DATA / "pysec-advisories_with_summary-expected.json"
         check_results_against_json(
             results=results,
             expected_file=expected_file,
@@ -32,12 +31,12 @@ def test_to_advisories_with_summary(self):
         )
 
     def test_to_advisories_without_summary(self):
-        with open(os.path.join(TEST_DATA, "pysec-advisories_without_summary.json")) as f:
+        with open(TEST_DATA / "pysec-advisories_without_summary.json") as f:
             mock_response = json.load(f)
 
         results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict()
 
-        expected_file = os.path.join(TEST_DATA, "pysec-advisories_without_summary-expected.json")
+        expected_file = TEST_DATA / "pysec-advisories_without_summary-expected.json"
         check_results_against_json(
             results=results,
             expected_file=expected_file,
@@ -45,14 +44,14 @@ def test_to_advisories_without_summary(self):
         )
 
     def test_to_advisories_with_cwe(self):
-        with open(os.path.join(TEST_DATA, "pysec-advisory_with_cwe.json")) as f:
+        with open(TEST_DATA / "pysec-advisory_with_cwe.json") as f:
             mock_response = json.load(f)
 
         results = parse_advisory_data(
             raw_data=mock_response, supported_ecosystems=["pypi"], advisory_url="https://tes.com"
         ).to_dict()
 
-        expected_file = os.path.join(TEST_DATA, "pysec-advisories_with_cwe-expected.json")
+        expected_file = TEST_DATA / "pysec-advisories_with_cwe-expected.json"
         check_results_against_json(
             results=results,
             expected_file=expected_file,

From cd9afea0b0088d0be5cf96f674070658a33cbaeb Mon Sep 17 00:00:00 2001
From: Keshav Priyadarshi <git@keshav.space>
Date: Thu, 24 Oct 2024 19:57:24 +0530
Subject: [PATCH 2/2] Add data migration for old pysec advisory

Signed-off-by: Keshav Priyadarshi <git@keshav.space>
---
 .../0074_update_pysec_advisory_created_by.py  | 37 ++++++++++++++
 vulnerabilities/pipelines/pysec_importer.py   |  3 +-
 vulnerabilities/tests/test_data_migrations.py | 49 +++++++++++++++++--
 3 files changed, 83 insertions(+), 6 deletions(-)
 create mode 100644 vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py

diff --git a/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py
new file mode 100644
index 000000000..d0e73181a
--- /dev/null
+++ b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py
@@ -0,0 +1,37 @@
+# Generated by Django 4.2.16 on 2024-10-24 13:51
+
+from django.db import migrations
+
+"""
+Update the created_by field on Advisory from the old qualified_name
+to the new pipeline_id.
+"""
+
+
+def update_created_by(apps, schema_editor):
+    from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline
+
+    Advisory = apps.get_model("vulnerabilities", "Advisory")
+    Advisory.objects.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").update(
+        created_by=PyPIImporterPipeline.pipeline_id
+    )
+
+
+def reverse_update_created_by(apps, schema_editor):
+    from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline
+
+    Advisory = apps.get_model("vulnerabilities", "Advisory")
+    Advisory.objects.filter(created_by=PyPIImporterPipeline.pipeline_id).update(
+        created_by="vulnerabilities.importers.pysec.PyPIImporter"
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0073_delete_packagerelatedvulnerability"),
+    ]
+
+    operations = [
+        migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by),
+    ]
diff --git a/vulnerabilities/pipelines/pysec_importer.py b/vulnerabilities/pipelines/pysec_importer.py
index c91e3be7b..32a9fd896 100644
--- a/vulnerabilities/pipelines/pysec_importer.py
+++ b/vulnerabilities/pipelines/pysec_importer.py
@@ -15,7 +15,6 @@
 import requests
 
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importers.osv import parse_advisory_data
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
 
 
@@ -48,6 +47,8 @@ def advisories_count(self) -> int:
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
         """Yield AdvisoryData using a zipped data dump of OSV data"""
+        from vulnerabilities.importers.osv import parse_advisory_data
+
         with ZipFile(BytesIO(self.advisory_zip)) as zip_file:
             for file_name in zip_file.namelist():
                 if not file_name.startswith("PYSEC-"):
diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py
index df51b12bb..046c86ce5 100644
--- a/vulnerabilities/tests/test_data_migrations.py
+++ b/vulnerabilities/tests/test_data_migrations.py
@@ -672,7 +672,7 @@ def setUpBeforeMigration(self, apps):
             date_collected=timezone.now(),
         )
 
-    def test_removal_of_duped_purls(self):
+    def test_update_npm_pypa_created_by_field(self):
         Advisory = apps.get_model("vulnerabilities", "Advisory")
         adv = Advisory.objects.all()
 
@@ -714,7 +714,7 @@ def setUpBeforeMigration(self, apps):
             date_collected=timezone.now(),
         )
 
-    def test_removal_of_duped_purls(self):
+    def test_update_nginx_created_by_field(self):
         Advisory = apps.get_model("vulnerabilities", "Advisory")
         adv = Advisory.objects.all()
 
@@ -753,7 +753,7 @@ def setUpBeforeMigration(self, apps):
             date_collected=timezone.now(),
         )
 
-    def test_removal_of_duped_purls(self):
+    def test_update_gitlab_created_by_field(self):
         Advisory = apps.get_model("vulnerabilities", "Advisory")
         adv = Advisory.objects.all()
 
@@ -794,7 +794,7 @@ def setUpBeforeMigration(self, apps):
             date_collected=timezone.now(),
         )
 
-    def test_removal_of_duped_purls(self):
+    def test_update_github_created_by_field(self):
         Advisory = apps.get_model("vulnerabilities", "Advisory")
         adv = Advisory.objects.all()
 
@@ -835,9 +835,48 @@ def setUpBeforeMigration(self, apps):
             date_collected=timezone.now(),
         )
 
-    def test_removal_of_duped_purls(self):
+    def test_update_nvd_created_by_field(self):
         Advisory = apps.get_model("vulnerabilities", "Advisory")
         adv = Advisory.objects.all()
 
         assert adv.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").count() == 0
         assert adv.filter(created_by="nvd_importer").count() == 1
+
+
+class TestUpdatePysecAdvisoryCreatedByField(TestMigrations):
+    app_name = "vulnerabilities"
+    migrate_from = "0073_delete_packagerelatedvulnerability"
+    migrate_to = "0074_update_pysec_advisory_created_by"
+
+    advisory_data1 = AdvisoryData(
+        aliases=["CVE-2020-13371337"],
+        summary="vulnerability description here",
+        affected_packages=[
+            AffectedPackage(
+                package=PackageURL(type="pypi", name="foobar"),
+                affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"),
+            )
+        ],
+        references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")],
+        date_published=timezone.now(),
+        url="https://test.com",
+    )
+
+    def setUpBeforeMigration(self, apps):
+        Advisory = apps.get_model("vulnerabilities", "Advisory")
+        adv1 = Advisory.objects.create(
+            aliases=self.advisory_data1.aliases,
+            summary=self.advisory_data1.summary,
+            affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages],
+            references=[ref.to_dict() for ref in self.advisory_data1.references],
+            url=self.advisory_data1.url,
+            created_by="vulnerabilities.importers.pysec.PyPIImporter",
+            date_collected=timezone.now(),
+        )
+
+    def test_update_pysec_created_by_field(self):
+        Advisory = apps.get_model("vulnerabilities", "Advisory")
+        adv = Advisory.objects.all()
+
+        assert adv.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").count() == 0
+        assert adv.filter(created_by="pysec_importer").count() == 1
