From f72d770feb4489d2f822a769f3509ea273848276 Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sat, 21 Nov 2020 11:24:41 +0530
Subject: [PATCH 1/9] Adapt rust importer to import data from new data format.

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/importers/rust.py | 36 +++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py
index f2a3e9997..61b450afe 100644
--- a/vulnerabilities/importers/rust.py
+++ b/vulnerabilities/importers/rust.py
@@ -29,7 +29,7 @@
 from urllib.error import HTTPError
 from urllib.request import urlopen
 
-import pytoml as toml
+import toml
 from dephell_specifier import RangeSpecifier
 from packageurl import PackageURL
 
@@ -47,7 +47,7 @@ def __enter__(self):
             self._added_files, self._updated_files = self.file_changes(
                 subdir="crates",  # TODO Consider importing the advisories for cargo, etc as well.
                 recursive=True,
-                file_ext="toml",
+                file_ext="md",
             )
 
     @property
@@ -66,15 +66,13 @@ def updated_advisories(self) -> Set[Advisory]:
         return self._load_advisories(self._updated_files)
 
     def _load_advisories(self, files) -> Set[Advisory]:
-        files = [f for f in files if not f.endswith("-0000.toml")]  # skip temporary files
+        files = [f for f in files if not f.endswith("-0000.md")]  # skip temporary files
         packages = self.collect_packages(files)
         self.set_api(packages)
 
         while files:
-            batch, files = files[: self.batch_size], files[self.batch_size:]
-
+            batch, files = files[: self.batch_size], files[self.batch_size :]
             advisories = set()
-
             for path in batch:
                 advisory = self._load_advisory(path)
                 if advisory:
@@ -84,13 +82,13 @@ def _load_advisories(self, files) -> Set[Advisory]:
     def collect_packages(self, paths):
         packages = set()
         for path in paths:
-            record = load_toml(path)
+            record = load_toml_from_md(path)
             packages.add(record["advisory"]["package"])
 
         return packages
 
     def _load_advisory(self, path: str) -> Optional[Advisory]:
-        record = load_toml(path)
+        record = load_toml_from_md(path)
         advisory = record.get("advisory", {})
         crate_name = advisory["package"]
         references = []
@@ -175,6 +173,22 @@ def categorize_versions(
     return unaffected, affected
 
 
-def load_toml(path):
-    with open(path) as f:
-        return toml.load(f)
+def load_toml_from_md(md_path):
+    # This collects the text between all the pairs of ```toml\n and ```\n characters
+    # and converts it into a parsed toml document and returns it.
+    parsed_data = {}
+    with open(md_path) as f:
+        lines = f.readlines()
+        for j, i in enumerate(lines):
+            if i == "```toml\n":
+                toml_lines = []
+                j += 1
+
+                while lines[j] != "```\n":
+                    toml_lines.append(lines[j])
+                    j += 1
+
+                parsed_toml = toml.loads("".join(toml_lines))
+                parsed_data.update(parsed_toml)
+
+    return parsed_data

From 18a274c32ad0a70b08c957dcfa973de082db8d6e Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sat, 21 Nov 2020 11:25:48 +0530
Subject: [PATCH 2/9] Upgrade tests for new rustsecdb format

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/package_managers.py           |   1 -
 .../tests/test_data/advisory-db.zip           | Bin 0 -> 527031 bytes
 .../tests/test_data/rust-advisory-db.zip      | Bin 527031 -> 901235 bytes
 vulnerabilities/tests/test_data_source.py     |   2 +-
 vulnerabilities/tests/test_rust.py            |  10 +++++-----
 5 files changed, 6 insertions(+), 7 deletions(-)
 create mode 100644 vulnerabilities/tests/test_data/advisory-db.zip

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 220bfb4d5..9e7c6e25d 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -319,5 +319,4 @@ async def fetch(self, owner_repo: str, session) -> None:
         endpoint = f"https://api.github.com/repos/{owner_repo}/git/refs/tags"
         resp = await session.request(method="GET", url=endpoint)
         resp = await resp.json()
-        print(resp)
         self.cache[owner_repo] = [release["ref"].split("/")[-1] for release in resp]
diff --git a/vulnerabilities/tests/test_data/advisory-db.zip b/vulnerabilities/tests/test_data/advisory-db.zip
new file mode 100644
index 0000000000000000000000000000000000000000..226a2d19a2ac0b02ecb79cd3f2bfcef21d5473fb
GIT binary patch
diff --git a/vulnerabilities/tests/test_data/rust-advisory-db.zip b/vulnerabilities/tests/test_data/rust-advisory-db.zip
index 226a2d19a2ac0b02ecb79cd3f2bfcef21d5473fb..cbc89af336ed16deac9250198f7e9033a8f98c71 100644
GIT binary patch
diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py
index 315c8e5c1..bbff49f99 100644
--- a/vulnerabilities/tests/test_data_source.py
+++ b/vulnerabilities/tests/test_data_source.py
@@ -268,7 +268,7 @@ class GitDataSourceTest(TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.tempdir = tempfile.mkdtemp()
-        zip_path = os.path.join(TEST_DATA, 'rust-advisory-db.zip')
+        zip_path = os.path.join(TEST_DATA, 'advisory-db.zip')
 
         with zipfile.ZipFile(zip_path, "r") as zip_ref:
             zip_ref.extractall(cls.tempdir)
diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py
index 5c90f2746..3d7c288ff 100644
--- a/vulnerabilities/tests/test_rust.py
+++ b/vulnerabilities/tests/test_rust.py
@@ -33,7 +33,7 @@
 from vulnerabilities import models
 from vulnerabilities.importers.rust import categorize_versions
 from vulnerabilities.import_runner import ImportRunner
-from vulnerabilities.package_managers import CratesVersionAPI
+from vulnerabilities.package_managers import VersionAPI
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, 'test_data/')
@@ -165,10 +165,10 @@ def test_import(self, _):
 
         with patch(
                 'vulnerabilities.importers.RustDataSource.crates_api',
-                new=CratesVersionAPI(cache=self.crates_api_cache)
-        ):
-            runner.run(cutoff_date=datetime.datetime(
-                year=2020, month=3, day=18, tzinfo=datetime.timezone.utc))
+                new=VersionAPI(cache=self.crates_api_cache)):
+            with patch('vulnerabilities.importers.RustDataSource.set_api'):
+                runner.run(cutoff_date=datetime.datetime(
+                    year=2020, month=3, day=18, tzinfo=datetime.timezone.utc))
 
         self.assert_for_package('bitvec', 'RUSTSEC-2020-0007')
         self.assert_for_package('bumpalo', 'RUSTSEC-2020-0006')

From 370fe2c8b90f33c30fcf7791302114eb0641f1ea Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sat, 21 Nov 2020 11:34:25 +0530
Subject: [PATCH 3/9] Update requirements.txt

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index e5d18cdb3..2653e5ed6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -56,3 +56,4 @@ wcwidth==0.1.7
 whitenoise==5.0.1
 zipp==0.6.0
 requests==2.23.0
+toml==0.10.2
\ No newline at end of file

From 44de8d2469547edca4e944e89491b48999b5d385 Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sat, 21 Nov 2020 11:40:00 +0530
Subject: [PATCH 4/9] Fix codestyle

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/importers/rust.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py
index 61b450afe..a2864b897 100644
--- a/vulnerabilities/importers/rust.py
+++ b/vulnerabilities/importers/rust.py
@@ -71,7 +71,7 @@ def _load_advisories(self, files) -> Set[Advisory]:
         self.set_api(packages)
 
         while files:
-            batch, files = files[: self.batch_size], files[self.batch_size :]
+            batch, files = files[: self.batch_size], files[self.batch_size:]
             advisories = set()
             for path in batch:
                 advisory = self._load_advisory(path)

From eb59d95dbb782ec3a061b2883f65a83e2129218b Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sun, 22 Nov 2020 14:18:21 +0530
Subject: [PATCH 5/9] Simplify rust importer's load_toml_from_md method by
 using regex

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/importers/rust.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py
index a2864b897..65eabdff5 100644
--- a/vulnerabilities/importers/rust.py
+++ b/vulnerabilities/importers/rust.py
@@ -23,6 +23,7 @@
 import asyncio
 import json
 from itertools import chain
+import re
 from typing import Optional, Mapping
 from typing import Set
 from typing import Tuple
@@ -174,21 +175,19 @@ def categorize_versions(
 
 
 def load_toml_from_md(md_path):
-    # This collects the text between all the pairs of ```toml\n and ```\n characters
-    # and converts it into a parsed toml document and returns it.
+    """
+    Return a mapping of vulnerability data from a RustSec advisory file at
+    ``location``.
+    RustSec advisories documents are .md files starting with a block of TOML
+    identified as the text inside a tripple-backtick TOML block. Per
+    https://github.com/RustSec/advisory-db#advisory-format:
+        Advisories are formatted in Markdown with TOML "front matter".
+    """
     parsed_data = {}
     with open(md_path) as f:
-        lines = f.readlines()
-        for j, i in enumerate(lines):
-            if i == "```toml\n":
-                toml_lines = []
-                j += 1
-
-                while lines[j] != "```\n":
-                    toml_lines.append(lines[j])
-                    j += 1
-
-                parsed_toml = toml.loads("".join(toml_lines))
-                parsed_data.update(parsed_toml)
-
+        result = re.findall("^```toml(.*?)^```", f.read(), re.DOTALL | re.MULTILINE)
+        # This regex matches everything that's between ```toml and ```
+        for i in result:
+            parsed_toml = toml.loads("".join(i))
+            parsed_data.update(parsed_toml)
     return parsed_data

From 76a316ebb38fc8558350aa8b46118fc41b95165b Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sun, 22 Nov 2020 14:19:36 +0530
Subject: [PATCH 6/9] Add more tests for rust importer

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 .../tests/test_data/rust/CVE-2019-16760.md    | 115 +++++++++++++++
 .../test_data/{ => rust}/rust-advisory-db.zip | Bin
 vulnerabilities/tests/test_rust.py            | 135 +++++++++++-------
 3 files changed, 196 insertions(+), 54 deletions(-)
 create mode 100644 vulnerabilities/tests/test_data/rust/CVE-2019-16760.md
 rename vulnerabilities/tests/test_data/{ => rust}/rust-advisory-db.zip (100%)

diff --git a/vulnerabilities/tests/test_data/rust/CVE-2019-16760.md b/vulnerabilities/tests/test_data/rust/CVE-2019-16760.md
new file mode 100644
index 000000000..fc6ad5bae
--- /dev/null
+++ b/vulnerabilities/tests/test_data/rust/CVE-2019-16760.md
@@ -0,0 +1,115 @@
+```toml
+[advisory]
+id = "CVE-2019-16760"
+package = "cargo"
+aliases = ["GHSA-phjm-8x66-qw4r"]
+date = "2019-09-30"
+url = "https://groups.google.com/forum/#!topic/rustlang-security-announcements/rVQ5e3TDnpQ"
+
+[versions]
+patched = [">= 1.26.0"]
+```
+
+# Cargo prior to Rust 1.26.0 may download the wrong dependency
+
+The Rust team was recently notified of a security concern when using older
+versions of Cargo to build crates which use the package rename feature added in
+newer versions of Cargo. If you're using Rust 1.26.0, released on 2018-05-10,
+or later you're not affected.
+
+The CVE for this vulnerability is [CVE-2019-16760][0].
+
+## Overview
+
+Cargo can be configured through `Cargo.toml` and the `[dependencies]` section
+to depend on different crates, such as those from crates.io. There are multiple
+ways to configure how you depend on crates as well, for example if you depend
+on `serde` and enable the `derive` feature it would look like:
+
+```toml
+serde = { version = "1.0", features = ['derive'] }
+```
+
+Rust 1.31.0 [introduced a new feature of Cargo][1] where one of the optional
+keys you can specify in this map is `package`, a way to [rename a crate
+locally][2]. For example if you preferred to use `serde1` locally instead of
+`serde`, you could write:
+
+```toml
+serde1 = { version = "1.0", features = ['derive'], package = "serde" }
+```
+
+It's the addition of the `package` key that causes Cargo to compile the crate
+differently. This feature was [first implemented][3] in Rust 1.26.0, but it was
+unstable at the time. For Rust 1.25.0 and prior, however, Cargo would ignore
+the `package` key and and interpret the dependency line as if it were:
+
+```toml
+serde1 = { version = "1.0", features = ['derive'] }
+```
+
+This means when compiled with Rust 1.25.0 and prior then it would attempt to
+download the `serde1` crate. A malicious user could squat the `serde1` name on
+crates.io to look like `serde 1.0.0` but instead act maliciously when built.
+
+In summary, usage of the `package` key to rename dependencies in `Cargo.toml`
+is ignored in Rust 1.25.0 and prior. When Rust 1.25.0 and prior is used Cargo
+will ignore `package` and download the wrong dependency, which could be
+squatted on crates.io to be a malicious package. This not only affects
+manifests that you write locally yourself, but also manifests published to
+crates.io. If you published a crate, for example, that depends on `serde1` to
+crates.io then users who depend on you may also be vulnerable if they use Rust
+1.25.0 and prior.
+
+## Affected Versions
+
+Rust 1.0.0 through Rust 1.25.0 is affected by this advisory because Cargo will
+ignore the `package` key in manifests. Rust 1.26.0 through Rust 1.30.0 are not
+affected and typically will emit an error because the `package` key is
+unstable. Rust 1.31.0 and after are not affected because Cargo understands the
+`package` key.
+
+In terms of Cargo versions, this affects Cargo up through Cargo 0.26.0. All
+future versions of Cargo are unaffected.
+
+## Mitigations
+
+We strongly recommend that users of the affected versions update their compiler
+to the latest available one. Preventing this issue from happening requires
+updating your compiler to either Rust 1.26.0 or newer.
+
+We will not be issuing a patch release for Rust versions prior to 1.26.0. Users
+of Rust 1.19.0 to Rust 1.25.0 can instead apply [the provided patches][4] to
+mitigate the issue.
+
+An audit of existing crates published to crates.io using the `package` key has
+been performed and there is no evidence that this vulnerability has been
+exploited in the wild. Our audit only covers the crates currently published on
+crates.io: if you notice crates exploiting this vulnerability in the future
+please don't hesitate to email secu...@rust-lang.org in accordance with [our
+security policy][5].
+
+## Timeline of events
+
+* Wed, Sep 18, 2019 at 13:54 UTC - Bug reported to secu...@rust-lang.org
+* Wed, Sep 18, 2019 at 15:35 UTC - Response confirming the report
+* Wed, Sep 18, 2019 - Cargo, Core, and crates.io teams confer on how best to
+handle this
+* Thu, Sep 19, 2019 - Confirmed with Elichai plan of action and continued to
+audit existing crates
+* Mon, Sep 23, 2019 - Advisory drafted, patches developed, audit completed
+* Mon, Sep 30, 2019 - Advisory published, security list informed of this issue
+
+## Acknowledgments
+
+Thanks to Elichai Turkel, who found this bug and reported it to us in accordance
+with our [security policy][5].
+
+## Links
+
+[0]: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-16760
+[1]: https://blog.rust-lang.org/2018/12/06/Rust-1.31-and-rust-2018.html#cargo-features
+[2]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml
+[3]: https://github.com/rust-lang/cargo/pull/4953
+[4]: https://gist.github.com/pietroalbini/0d293b24a44babbeb6187e06eebd4992
+[5]: https://www.rust-lang.org/policies/security
diff --git a/vulnerabilities/tests/test_data/rust-advisory-db.zip b/vulnerabilities/tests/test_data/rust/rust-advisory-db.zip
similarity index 100%
rename from vulnerabilities/tests/test_data/rust-advisory-db.zip
rename to vulnerabilities/tests/test_data/rust/rust-advisory-db.zip
diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py
index 3d7c288ff..754d8ba9e 100644
--- a/vulnerabilities/tests/test_rust.py
+++ b/vulnerabilities/tests/test_rust.py
@@ -34,25 +34,26 @@
 from vulnerabilities.importers.rust import categorize_versions
 from vulnerabilities.import_runner import ImportRunner
 from vulnerabilities.package_managers import VersionAPI
+from vulnerabilities.importers.rust import load_toml_from_md
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_DATA = os.path.join(BASE_DIR, 'test_data/')
+TEST_DATA = os.path.join(BASE_DIR, "test_data/rust")
 
 MOCKED_CRATES_API_VERSIONS = {
-    'bitvec': ['0.10.0', '0.12.0', '0.18.0'],
-    'bumpalo': ['2.8.0', '3.0.1', '3.2.5'],
-    'cbox': ['0.10.0', '0.12.0', '0.18.0'],
-    'flatbuffers': ['0.3.0', '0.5.0', '0.6.5'],
-    'hyper': ['0.10.0', '0.12.0', '0.13.0'],
+    "bitvec": ["0.10.0", "0.12.0", "0.18.0"],
+    "bumpalo": ["2.8.0", "3.0.1", "3.2.5"],
+    "cbox": ["0.10.0", "0.12.0", "0.18.0"],
+    "flatbuffers": ["0.3.0", "0.5.0", "0.6.5"],
+    "hyper": ["0.10.0", "0.12.0", "0.13.0"],
 }
 
 
 def test_categorize_versions():
-    flatbuffers_versions = MOCKED_CRATES_API_VERSIONS['flatbuffers']
+    flatbuffers_versions = MOCKED_CRATES_API_VERSIONS["flatbuffers"]
 
-    unaffected_ranges = {RangeSpecifier('< 0.4.0')}
-    affected_ranges = {RangeSpecifier('>= 0.4.0'), RangeSpecifier('<= 0.6.0')}
-    resolved_ranges = {RangeSpecifier('>= 0.6.1')}
+    unaffected_ranges = {RangeSpecifier("< 0.4.0")}
+    affected_ranges = {RangeSpecifier(">= 0.4.0"), RangeSpecifier("<= 0.6.0")}
+    resolved_ranges = {RangeSpecifier(">= 0.6.1")}
 
     unaffected_versions, affected_versions = categorize_versions(
         set(flatbuffers_versions),
@@ -62,19 +63,19 @@ def test_categorize_versions():
     )
 
     assert len(unaffected_versions) == 2
-    assert '0.3.0' in unaffected_versions
-    assert '0.6.5' in unaffected_versions
+    assert "0.3.0" in unaffected_versions
+    assert "0.6.5" in unaffected_versions
 
     assert len(affected_versions) == 1
-    assert '0.5.0' in affected_versions
+    assert "0.5.0" in affected_versions
 
 
 def test_categorize_versions_without_affected_ranges():
-    all_versions = {'1.0', '1.1', '2.0', '2.1', '3.0', '3.1'}
+    all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"}
 
-    unaffected_ranges = {RangeSpecifier('< 1.2')}
+    unaffected_ranges = {RangeSpecifier("< 1.2")}
     affected_ranges = set()
-    resolved_ranges = {RangeSpecifier('>= 3.0')}
+    resolved_ranges = {RangeSpecifier(">= 3.0")}
 
     unaffected_versions, affected_versions = categorize_versions(
         all_versions,
@@ -84,21 +85,21 @@ def test_categorize_versions_without_affected_ranges():
     )
 
     assert len(unaffected_versions) == 4
-    assert '1.0' in unaffected_versions
-    assert '1.1' in unaffected_versions
-    assert '3.0' in unaffected_versions
-    assert '3.1' in unaffected_versions
+    assert "1.0" in unaffected_versions
+    assert "1.1" in unaffected_versions
+    assert "3.0" in unaffected_versions
+    assert "3.1" in unaffected_versions
 
     assert len(affected_versions) == 2
-    assert '2.0' in affected_versions
-    assert '2.1' in affected_versions
+    assert "2.0" in affected_versions
+    assert "2.1" in affected_versions
 
 
 def test_categorize_versions_with_only_affected_ranges():
-    all_versions = {'1.0', '1.1', '2.0', '2.1', '3.0', '3.1'}
+    all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"}
 
     unaffected_ranges = set()
-    affected_ranges = {RangeSpecifier('> 1.2'), RangeSpecifier('<= 2.1')}
+    affected_ranges = {RangeSpecifier("> 1.2"), RangeSpecifier("<= 2.1")}
     resolved_ranges = set()
 
     unaffected_versions, affected_versions = categorize_versions(
@@ -109,18 +110,18 @@ def test_categorize_versions_with_only_affected_ranges():
     )
 
     assert len(unaffected_versions) == 4
-    assert '1.0' in unaffected_versions
-    assert '1.1' in unaffected_versions
-    assert '3.0' in unaffected_versions
-    assert '3.1' in unaffected_versions
+    assert "1.0" in unaffected_versions
+    assert "1.1" in unaffected_versions
+    assert "3.0" in unaffected_versions
+    assert "3.1" in unaffected_versions
 
     assert len(affected_versions) == 2
-    assert '2.0' in affected_versions
-    assert '2.1' in affected_versions
+    assert "2.0" in affected_versions
+    assert "2.1" in affected_versions
 
 
 def test_categorize_versions_without_any_ranges():
-    all_versions = {'1.0', '1.1', '2.0', '2.1', '3.0', '3.1'}
+    all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"}
 
     unaffected, affected = categorize_versions(all_versions, set(), set(), set())
 
@@ -128,7 +129,7 @@ def test_categorize_versions_without_any_ranges():
     assert len(affected) == 0
 
 
-@patch('vulnerabilities.importers.RustDataSource._update_from_remote')
+@patch("vulnerabilities.importers.RustDataSource._update_from_remote")
 class RustImportTest(TestCase):
 
     tempdir = None
@@ -136,21 +137,21 @@ class RustImportTest(TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.tempdir = tempfile.mkdtemp()
-        zip_path = os.path.join(TEST_DATA, 'rust-advisory-db.zip')
+        zip_path = os.path.join(TEST_DATA, "rust-advisory-db.zip")
 
         with zipfile.ZipFile(zip_path, "r") as zip_ref:
             zip_ref.extractall(cls.tempdir)
 
         cls.importer = models.Importer.objects.create(
-            name='rust_unittests',
-            license='https://creativecommons.org/publicdomain/zero/1.0/',
+            name="rust_unittests",
+            license="https://creativecommons.org/publicdomain/zero/1.0/",
             last_run=None,
-            data_source='RustDataSource',
+            data_source="RustDataSource",
             data_source_cfg={
-                'repository_url': 'https://example.com/unit-tests/advisory-db',
-                'working_directory': os.path.join(cls.tempdir, 'advisory-db'),
-                'create_working_directory': False,
-                'remove_working_directory': False,
+                "repository_url": "https://example.com/unit-tests/advisory-db",
+                "working_directory": os.path.join(cls.tempdir, "advisory-db"),
+                "create_working_directory": False,
+                "remove_working_directory": False,
             },
         )
 
@@ -164,20 +165,43 @@ def test_import(self, _):
         runner = ImportRunner(self.importer, 5)
 
         with patch(
-                'vulnerabilities.importers.RustDataSource.crates_api',
-                new=VersionAPI(cache=self.crates_api_cache)):
-            with patch('vulnerabilities.importers.RustDataSource.set_api'):
-                runner.run(cutoff_date=datetime.datetime(
-                    year=2020, month=3, day=18, tzinfo=datetime.timezone.utc))
-
-        self.assert_for_package('bitvec', 'RUSTSEC-2020-0007')
-        self.assert_for_package('bumpalo', 'RUSTSEC-2020-0006')
-        self.assert_for_package('flatbuffers', 'RUSTSEC-2019-0028')
-        self.assert_for_package('hyper', 'RUSTSEC-2020-0008')
+            "vulnerabilities.importers.RustDataSource.crates_api",
+            new=VersionAPI(cache=self.crates_api_cache),
+        ):
+            with patch("vulnerabilities.importers.RustDataSource.set_api"):
+                runner.run(
+                    cutoff_date=datetime.datetime(
+                        year=2020, month=3, day=18, tzinfo=datetime.timezone.utc
+                    )
+                )
+
+        self.assert_for_package("bitvec", "RUSTSEC-2020-0007")
+        self.assert_for_package("bumpalo", "RUSTSEC-2020-0006")
+        self.assert_for_package("flatbuffers", "RUSTSEC-2019-0028")
+        self.assert_for_package("hyper", "RUSTSEC-2020-0008")
 
         # There is no data for cbox, because the advisory contains neither affected nor patched or
         # unaffected versions.
-        assert models.Package.objects.filter(name='cbox').count() == 0
+        assert models.Package.objects.filter(name="cbox").count() == 0
+
+    def test_load_toml_from_md(self, _):
+        md_path = os.path.join(TEST_DATA, "CVE-2019-16760.md")
+        # print(list(os.walk(self.tempdir)))
+        loaded_data = load_toml_from_md(md_path)
+        expected_data = {
+            "advisory": {
+                "aliases": ["GHSA-phjm-8x66-qw4r"],
+                "date": "2019-09-30",
+                "id": "CVE-2019-16760",
+                "package": "cargo",
+                "url": "https://groups.google.com/forum/#!topic/rustlang-security-announcements/rVQ5e3TDnpQ",  # nopep8
+            },
+            "serde": {"features": ["derive"], "version": "1.0"},
+            "serde1": {"features": ["derive"], "version": "1.0"},
+            "versions": {"patched": [">= 1.26.0"]},
+        }
+
+        assert loaded_data == expected_data
 
     def assert_for_package(self, package, advisory_id):
         qs = models.Package.objects.filter(name=package)
@@ -192,12 +216,15 @@ def assert_for_package(self, package, advisory_id):
         vuln = qs[0].vulnerability
 
         assert models.PackageRelatedVulnerability.objects.filter(
-            package=unaffected_pkg, vulnerability=vuln, is_vulnerable=False)
+            package=unaffected_pkg, vulnerability=vuln, is_vulnerable=False
+        )
 
         assert models.PackageRelatedVulnerability.objects.filter(
-            package=resolved_pkg, vulnerability=vuln, is_vulnerable=False)
+            package=resolved_pkg, vulnerability=vuln, is_vulnerable=False
+        )
 
         assert models.PackageRelatedVulnerability.objects.filter(
-            package=impacted_pkg, vulnerability=vuln, is_vulnerable=True)
+            package=impacted_pkg, vulnerability=vuln, is_vulnerable=True
+        )
 
     importer = None

From 08617de75fd25c56f3868a697ab4ab70e0bb4729 Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Mon, 23 Nov 2020 20:55:17 +0530
Subject: [PATCH 7/9] Break load_toml_from_md into smaller methods

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/importers/rust.py  | 65 +++++++++++++++++++++++++-----
 vulnerabilities/tests/test_rust.py |  6 +--
 2 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py
index 65eabdff5..a974ae70e 100644
--- a/vulnerabilities/importers/rust.py
+++ b/vulnerabilities/importers/rust.py
@@ -83,13 +83,13 @@ def _load_advisories(self, files) -> Set[Advisory]:
     def collect_packages(self, paths):
         packages = set()
         for path in paths:
-            record = load_toml_from_md(path)
+            record = get_advisory_data(path)
             packages.add(record["advisory"]["package"])
 
         return packages
 
     def _load_advisory(self, path: str) -> Optional[Advisory]:
-        record = load_toml_from_md(path)
+        record = get_advisory_data(path)
         advisory = record.get("advisory", {})
         crate_name = advisory["package"]
         references = []
@@ -174,7 +174,54 @@ def categorize_versions(
     return unaffected, affected
 
 
-def load_toml_from_md(md_path):
+def get_toml_lines(lines):
+    """
+    Yield lines of TOML extracted from an iterable of text ``lines``.
+    The lines are expected to be from a RustSec Markdown advisory file with
+    embedded TOML metadata.
+
+    For example::
+
+    >>> text = '''
+    ... ```toml
+    ... [advisory]
+    ... id = "RUST-001"
+    ...
+    ... [versions]
+    ... patch = [">= 1.2.1"]
+    ... ```
+    ... # Use-after-free with objects returned by `Stream`'s `get_format_info`
+    ...
+    ... Affected versions contained a pair of use-after-free issues with the objects.
+    ... '''
+    >>> list(get_toml_lines(text.splitlines()))
+    ['', '[advisory]', 'id = "RUST-001"', '', '[versions]', 'patch = [">= 1.2.1"]']
+    """
+
+    for line in lines:
+        line = line.strip()
+        if line.startswith("```toml"):
+            continue
+        elif line.endswith("```"):
+            break
+        else:
+            yield line
+
+
+def data_from_toml_lines(lines):
+    """
+    Return a mapping of data from an iterable of TOML text ``lines``.
+
+    For example::
+
+    >>> lines = ['[advisory]', 'id = "RUST1"', '', '[versions]', 'patch = [">= 1"]']
+    >>> data_from_toml_lines(lines)
+    {'advisory': {'id': 'RUST1'}, 'versions': {'patch': ['>= 1']}}
+    """
+    return toml.loads("\n".join(lines))
+
+
+def get_advisory_data(location):
     """
     Return a mapping of vulnerability data from a RustSec advisory file at
     ``location``.
@@ -183,11 +230,7 @@ def load_toml_from_md(md_path):
     https://github.com/RustSec/advisory-db#advisory-format:
         Advisories are formatted in Markdown with TOML "front matter".
     """
-    parsed_data = {}
-    with open(md_path) as f:
-        result = re.findall("^```toml(.*?)^```", f.read(), re.DOTALL | re.MULTILINE)
-        # This regex matches everything that's between ```toml and ```
-        for i in result:
-            parsed_toml = toml.loads("".join(i))
-            parsed_data.update(parsed_toml)
-    return parsed_data
+
+    with open(location) as lines:
+        toml_lines = get_toml_lines(lines)
+        return data_from_toml_lines(toml_lines)
diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py
index 754d8ba9e..5252f5b58 100644
--- a/vulnerabilities/tests/test_rust.py
+++ b/vulnerabilities/tests/test_rust.py
@@ -34,7 +34,7 @@
 from vulnerabilities.importers.rust import categorize_versions
 from vulnerabilities.import_runner import ImportRunner
 from vulnerabilities.package_managers import VersionAPI
-from vulnerabilities.importers.rust import load_toml_from_md
+from vulnerabilities.importers.rust import get_advisory_data
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, "test_data/rust")
@@ -187,7 +187,7 @@ def test_import(self, _):
     def test_load_toml_from_md(self, _):
         md_path = os.path.join(TEST_DATA, "CVE-2019-16760.md")
         # print(list(os.walk(self.tempdir)))
-        loaded_data = load_toml_from_md(md_path)
+        loaded_data = get_advisory_data(md_path)
         expected_data = {
             "advisory": {
                 "aliases": ["GHSA-phjm-8x66-qw4r"],
@@ -196,8 +196,6 @@ def test_load_toml_from_md(self, _):
                 "package": "cargo",
                 "url": "https://groups.google.com/forum/#!topic/rustlang-security-announcements/rVQ5e3TDnpQ",  # nopep8
             },
-            "serde": {"features": ["derive"], "version": "1.0"},
-            "serde1": {"features": ["derive"], "version": "1.0"},
             "versions": {"patched": [">= 1.26.0"]},
         }
 

From 03593cd8b098ca91d9e5e5b9ac6d1f19158953a5 Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Wed, 25 Nov 2020 15:59:45 +0530
Subject: [PATCH 8/9] Ignore collecting temporary vulnerabilities from
 debian.py

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/importers/debian.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py
index bfacae1c5..e616a834b 100644
--- a/vulnerabilities/importers/debian.py
+++ b/vulnerabilities/importers/debian.py
@@ -108,6 +108,8 @@ def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[Advisory]:
 
         for cve_id, record in records.items():
             impacted_purls, resolved_purls = set(), set()
+            if not cve_id.startswith("CVE"):
+                continue
 
             for release_name, release_record in record["releases"].items():
                 if not release_record.get("repositories", {}).get(release_name):

From 155488984253effd8ac8982fafae92453563014f Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Wed, 25 Nov 2020 16:00:37 +0530
Subject: [PATCH 9/9] Add tests to check if debian.py imports temporary
 vulnerabilities

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
---
 vulnerabilities/tests/test_data/debian.json | 23 +++++++++++++++++++++
 vulnerabilities/tests/test_debian.py        |  1 +
 2 files changed, 24 insertions(+)

diff --git a/vulnerabilities/tests/test_data/debian.json b/vulnerabilities/tests/test_data/debian.json
index da8e78fcd..bcd81d076 100644
--- a/vulnerabilities/tests/test_data/debian.json
+++ b/vulnerabilities/tests/test_data/debian.json
@@ -72,6 +72,29 @@
           "urgency": "medium"
         }
       }
+    },
+    "TEMP-0000000-A2EB44": {
+      "scope": "remote",
+      "debianbug": 456,
+      "description": "Multiple temp vulnerabilities in foo bar",
+      "releases": {
+        "stretch": {
+          "status": "resolved",
+          "repositories": {
+            "stretch": "1.2.3"
+          },
+          "fixed_version": "1.3.3",
+          "urgency": "medium"
+        },
+        "buster": {
+          "status": "resolved",
+          "repositories": {
+            "buster": "1.3.4"
+          },
+          "fixed_version": "1.4.5",
+          "urgency": "medium"
+        }
+      }
     }
   }
 }
diff --git a/vulnerabilities/tests/test_debian.py b/vulnerabilities/tests/test_debian.py
index f86285544..e35f17761 100644
--- a/vulnerabilities/tests/test_debian.py
+++ b/vulnerabilities/tests/test_debian.py
@@ -77,6 +77,7 @@ def test_import(self):
         self.assert_for_package("mimetex", "1.74-1", "stretch")
         self.assert_for_package("mimetex", "1.50-1.1", "buster")
         self.assert_for_package("mimetex", "1.76-1", "buster")
+        assert models.Vulnerability.objects.filter(cve_id__startswith="TEMP").count() == 0
 
     def test_response_is_new(self):
 
