Dummy branch

.github/workflows/main.yml

@@ -0,0 +1,48 @@
+name: CI
+on:
+  pull_request:
+    types: [opened, synchronize]
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+
+      - name: Run tests
+        run: pytest
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 

pyproject.toml

@@ -43,4 +43,5 @@ addopts = [
     "-rfExXw",
     "--strict",
     "--doctest-modules"
-]
+]
+

requirements.txt

@@ -42,10 +42,7 @@ pytest-dependency==0.4.0
 pytest-django==3.7.0
 pytest-mock==1.13.0
 python-dateutil==2.8.1
-pytoml==0.1.21
 pytz==2019.3
-PyYAML==5.3.1
-saneyaml==0.4
 schema==0.7.1
 six==1.13.0
 soupsieve==1.9.5
@@ -56,3 +53,5 @@ wcwidth==0.1.7
 whitenoise==5.0.1
 zipp==0.6.0
 requests==2.23.0
+toml==0.10.2
+PyYAML==5.3.1

vulnerabilities/helpers.py

@@ -0,0 +1,67 @@
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import json
+import re
+
+import yaml
+import requests
+import toml
+
+
+def load_yaml(path):
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def load_json(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+def load_toml(path):
+    with open(path) as f:
+        return toml.load(f)
+
+
+def create_etag(data_src, url, etag_key):
+    """
+    Etags are like hashes of web responses. For a data source `data_src`,
+    we maintain (url, etag) mappings in the DB.  `create_etag`  creates
+    (`url`, etag) pair. If a (`url`, etag) already exists then the code
+    skips processing the response further to avoid duplicate work.
+
+    `etag_key` is the name of header which contains the etag for the url.
+    """
+    etag = requests.head(url).headers.get(etag_key)
+    if not etag:
+        return True
+
+    elif url in data_src.config.etags:
+        if data_src.config.etags[url] == etag:
+            return False
+
+    data_src.config.etags[url] = etag
+    return True
+
+
+is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match

vulnerabilities/import_runner.py

@@ -90,7 +90,7 @@ class ImportRunner:
     Efficiency:
         - Bulk inserts should be used whenever possible.
         - Checking whether a record already exists should be kept to a minimum
-          (the data source should know this instead).
+        (the data source should know this instead).
         - All update and select operations must use indexed columns.
     """
 

vulnerabilities/importer_yielder.py

@@ -188,7 +188,7 @@
         'last_run': None,
         'data_source': 'ProjectKBMSRDataSource',
         'data_source_cfg': {
-            'etag': {}
+            'etags': {}
         }
     },
     {
@@ -216,7 +216,7 @@
         'last_run': None,
         'data_source': 'NginxDataSource',
         'data_source_cfg': {
-            'etag': {}
+            'etags': {}
         },
     },
     {

vulnerabilities/importers/alpine_linux.py

@@ -20,13 +20,13 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
 from typing import Any
 from typing import Iterable
 from typing import List
 from typing import Mapping
 from typing import Set
 
-import yaml
 from packageurl import PackageURL
 from schema import Or
 from schema import Regex
@@ -35,6 +35,7 @@
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import GitDataSource
 from vulnerabilities.data_source import Reference
+from vulnerabilities.helpers import load_yaml
 
 
 def validate_schema(advisory_dict):
@@ -90,19 +91,17 @@ def updated_advisories(self) -> Set[Advisory]:
     def _process_file(self, path) -> List[Advisory]:
         advisories = []
 
-        with open(path) as f:
-            record = yaml.safe_load(f)
-
-            if record["packages"] is None:
-                return advisories
-            validate_schema(record)
+        record = load_yaml(path)
+        if record["packages"] is None:
+            return advisories
+        validate_schema(record)
 
-            for p in record["packages"]:
-                advisories.extend(
-                    self._load_advisories(
-                        p["pkg"], record["distroversion"], record["reponame"], record["archs"],
-                    )
+        for p in record["packages"]:
+            advisories.extend(
+                self._load_advisories(
+                    p["pkg"], record["distroversion"], record["reponame"], record["archs"],
                 )
+            )
 
         return advisories
 

vulnerabilities/importers/apache_httpd.py

@@ -20,7 +20,7 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from dataclasses import dataclass
+import dataclasses
 from xml.etree import ElementTree
 
 import requests
@@ -29,9 +29,10 @@
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import DataSourceConfiguration
+from vulnerabilities.helpers import create_etag
 
 
-@dataclass
+@dataclasses.dataclass
 class ApacheHTTPDDataSourceConfiguration(DataSourceConfiguration):
     etags: dict
 
@@ -47,25 +48,13 @@ def updated_advisories(self):
         # (url, etag) pair. If a (url, etag) already exists then the code
         # skips processing the response further to avoid duplicate work
 
-        if self.create_etag(self.url):
+        if create_etag(data_src=self, url=self.url, etag_key="ETag"):
             data = fetch_xml(self.url)
             advisories = to_advisories(data)
             return self.batch_advisories(advisories)
 
         return []
 
-    def create_etag(self, url):
-        etag = requests.head(url).headers.get("ETag")
-        if not etag:
-            return True
-
-        elif url in self.config.etags:
-            if self.config.etags[url] == etag:
-                return False
-
-        self.config.etags[url] = etag
-        return True
-
 
 def to_advisories(data):
     advisories = []

vulnerabilities/importers/debian_oval.py

@@ -35,6 +35,7 @@
 
 from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration
 from vulnerabilities.package_managers import DebianVersionAPI
+from vulnerabilities.helpers import create_etag
 
 
 @dataclasses.dataclass
@@ -52,35 +53,22 @@ def __init__(self, *args, **kwargs):
         # we could avoid setting translations, and have it
         # set by default in the OvalParser, but we don't yet know
         # whether all OVAL providers use the same format
-        self.translations = {'less than': '<'}
+        self.translations = {"less than": "<"}
         self.pkg_manager_api = DebianVersionAPI()
 
     def _fetch(self):
-        base_url = 'https://www.debian.org/security/oval/'
-        file_name = 'oval-definitions-{}.xml'
         releases = self.config.releases
         for release in releases:
-            file_url = base_url + file_name.format(release)
-            if not self.create_etag(file_url):
+            file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml"
+            if not create_etag(data_src=self, url=file_url, etag_key="ETag"):
                 continue
+
             resp = requests.get(file_url).content
             yield (
-                {'type': 'deb', 'namespace': 'debian',
-                    'qualifiers': {'distro': release}
-                 },
-                ET.ElementTree(ET.fromstring(resp.decode('utf-8')))
+                {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}},
+                ET.ElementTree(ET.fromstring(resp.decode("utf-8"))),
             )
         return []
 
     def set_api(self, packages):
         asyncio.run(self.pkg_manager_api.load_api(packages))
-
-    def create_etag(self, url):
-        etag = requests.head(url).headers.get('ETag')
-        if not etag:
-            return True
-        elif url in self.config.etags:
-            if self.config.etags[url] == etag:
-                return False
-        self.config.etags[url] = etag
-        return True

vulnerabilities/importers/kaybee.py

@@ -20,13 +20,12 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-import yaml
-
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import GitDataSource
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import Reference
+from vulnerabilities.helpers import load_yaml
 
 
 class KaybeeDataSource(GitDataSource):
@@ -36,14 +35,11 @@ def __enter__(self):
             recursive=True,
             file_ext="yaml",
         )
-        print(self._added_files.union(self._updated_files))
 
     def updated_advisories(self):
         advisories = []
         for yaml_file in self._added_files.union(self._updated_files):
-            print(yaml_file)
             advisories.append(yaml_file_to_advisory(yaml_file))
-            print(advisories[-1])
 
         return self.batch_advisories(advisories)
 
@@ -76,9 +72,3 @@ def yaml_file_to_advisory(yaml_path):
         resolved_package_urls=resolved_packages,
         vuln_references=references,
     )
-
-
-# TODO refactor all such  commonly needed helpers into one single module
-def load_yaml(path):
-    with open(path) as f:
-        return yaml.safe_load(f)