diff --git a/cyclonedx_py/parser/poetry.py b/cyclonedx_py/parser/poetry.py index 4545f5ac..e3921e08 100644 --- a/cyclonedx_py/parser/poetry.py +++ b/cyclonedx_py/parser/poetry.py @@ -60,4 +60,3 @@ class PoetryFileParser(PoetryParser): def __init__(self, poetry_lock_filename: str) -> None: with open(poetry_lock_filename) as r: super(PoetryFileParser, self).__init__(poetry_lock_contents=r.read()) - r.close() diff --git a/cyclonedx_py/parser/requirements.py b/cyclonedx_py/parser/requirements.py index 520eb3b3..740a5b1a 100644 --- a/cyclonedx_py/parser/requirements.py +++ b/cyclonedx_py/parser/requirements.py @@ -17,49 +17,64 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) OWASP Foundation. All Rights Reserved. +import os +import os.path +from tempfile import NamedTemporaryFile, _TemporaryFileWrapper # Weak error +from typing import Any, Optional + +from cyclonedx.model import HashType from cyclonedx.model.component import Component from cyclonedx.parser import BaseParser, ParserWarning # See https://github.com/package-url/packageurl-python/issues/65 from packageurl import PackageURL # type: ignore -from pkg_resources import parse_requirements as parse_requirements +from pip_requirements_parser import RequirementsFile # type: ignore class RequirementsParser(BaseParser): def __init__(self, requirements_content: str) -> None: super().__init__() + parsed_rf: Optional[RequirementsFile] = None + requirements_file: Optional[_TemporaryFileWrapper[Any]] = None - requirements = parse_requirements(requirements_content) - for requirement in requirements: - """ - @todo - Note that the below line will get the first (lowest) version specified in the Requirement and - ignore the operator (it might not be ==). This is passed to the Component. + if os.path.exists(requirements_content): + parsed_rf = RequirementsFile.from_file( + requirements_content, include_nested=True) + else: + requirements_file = NamedTemporaryFile(mode='w+', delete=False) + requirements_file.write(requirements_content) + requirements_file.close() - For example if a requirement was listed as: "PickyThing>1.6,<=1.9,!=1.8.6", we'll be interpreting this - as if it were written "PickyThing==1.6" - """ - try: - (op, version) = requirement.specs[0] - self._components.append(Component( - name=requirement.project_name, version=version, purl=PackageURL( - type='pypi', name=requirement.project_name, version=version - ) - )) - except IndexError: + parsed_rf = RequirementsFile.from_file( + requirements_file.name, include_nested=False) + + for requirement in parsed_rf.requirements: + name = requirement.link.url if requirement.is_local_path else requirement.name + version = requirement.get_pinned_version or requirement.dumps_specifier() + hashes = map(HashType.from_composite_str, requirement.hash_options) + + if not version and not requirement.is_local_path: self._warnings.append( ParserWarning( - item=requirement.project_name, - warning='Requirement \'{}\' does not have a pinned version and cannot be included in your ' - 'CycloneDX SBOM.'.format(requirement.project_name) + item=name, + warning=(f"Requirement \'{name}\' does not have a pinned " + "version and cannot be included in your CycloneDX SBOM.") ) ) + else: + self._components.append(Component( + name=name, + version=version, + hashes=hashes, + purl=PackageURL(type='pypi', name=name, version=version) + )) + + if requirements_file: + os.unlink(requirements_file.name) class RequirementsFileParser(RequirementsParser): def __init__(self, requirements_file: str) -> None: - with open(requirements_file) as r: - super(RequirementsFileParser, self).__init__(requirements_content=r.read()) - r.close() + super().__init__(requirements_content=requirements_file) diff --git a/docs/usage.rst b/docs/usage.rst index 3e23a991..fdac4530 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -215,8 +215,8 @@ Requirements * :py:mod:`cyclonedx_py.parser.requirements.RequirementsParser`: Parses a multiline string that you provide that conforms to the ``requirements.txt`` :pep:`508` standard. -* :py:mod:`cyclonedx_py.parser.requirements.RequirementsFileParser`: Parses a file that you provide the path to that - conforms to the ``requirements.txt`` :pep:`508` standard. +* :py:mod:`cyclonedx_py.parser.requirements.RequirementsFileParser`: Parses a file that you provide the path to that conforms to the ``requirements.txt`` :pep:`508` standard. It supports nested +files, so if there is a line in your ``requirements.txt`` file with the ``-r requirements-nested.txt`` syntax, it'll parse the nested file as part of the same file. CycloneDX software bill-of-materials require pinned versions of requirements. If your `requirements.txt` does not have pinned versions, warnings will be recorded and the dependencies without pinned versions will be excluded from the diff --git a/poetry.lock b/poetry.lock index fedb94ef..ecb3ee80 100644 --- a/poetry.lock +++ b/poetry.lock @@ -230,13 +230,28 @@ test = ["isort", "pytest"] name = "packaging" version = "21.3" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +[[package]] +name = "pip-requirements-parser" +version = "31.2.0" +description = "pip requirements parser - a mostly correct pip requirements parsing library because it uses pip's own code." +category = "main" +optional = false +python-versions = ">=3.6.*" + +[package.dependencies] +packaging = "*" + +[package.extras] +docs = ["Sphinx (>=3.3.1)", "sphinx-rtd-theme (>=0.5.0)", "doc8 (>=0.8.1)"] +testing = ["pytest (>=6)", "pytest-xdist (>=2)"] + [[package]] name = "platformdirs" version = "2.4.0" @@ -292,7 +307,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "pyparsing" version = "3.0.7" description = "Python parsing module" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -309,7 +324,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "testfixtures" -version = "6.18.4" +version = "6.18.5" description = "A collection of helpers and mock objects for unit tests and doc tests." category = "dev" optional = false @@ -369,7 +384,7 @@ python-versions = ">=3.6" [[package]] name = "types-setuptools" -version = "57.4.9" +version = "57.4.10" description = "Typing stubs for setuptools" category = "main" optional = false @@ -393,7 +408,7 @@ python-versions = "*" [[package]] name = "virtualenv" -version = "20.13.2" +version = "20.13.3" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -426,7 +441,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "3ab9c98733316487bfc14e06b837c17a5b7680e31ec9212100dec921ddee992a" +content-hash = "3d849eb2deecacf30c555144409a871b755f0a83dcb8b6de9c87df29f2146244" [metadata.files] attrs = [ @@ -568,6 +583,10 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] +pip-requirements-parser = [ + {file = "pip-requirements-parser-31.2.0.tar.gz", hash = "sha256:8c2a6f8e091ac2693824a5ef4e3b250226e34f74a20a91a87b9ab0714b47788f"}, + {file = "pip_requirements_parser-31.2.0-py3-none-any.whl", hash = "sha256:22fa213a987913385b2484d5698ecfa1d9cf4154978cdf929085548af55355b0"}, +] platformdirs = [ {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, @@ -597,8 +616,8 @@ six = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] testfixtures = [ - {file = "testfixtures-6.18.4-py2.py3-none-any.whl", hash = "sha256:27cfa35006407ef31a4e8752873ca2232fd5761e29047a86c919e363e0e17196"}, - {file = "testfixtures-6.18.4.tar.gz", hash = "sha256:878f617c411793f155c26b39330a095a1cc58844d88bcdc767a65c7fc2096b54"}, + {file = "testfixtures-6.18.5-py2.py3-none-any.whl", hash = "sha256:7de200e24f50a4a5d6da7019fb1197aaf5abd475efb2ec2422fdcf2f2eb98c1d"}, + {file = "testfixtures-6.18.5.tar.gz", hash = "sha256:02dae883f567f5b70fd3ad3c9eefb95912e78ac90be6c7444b5e2f46bf572c84"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, @@ -639,8 +658,8 @@ typed-ast = [ {file = "typed_ast-1.5.2.tar.gz", hash = "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27"}, ] types-setuptools = [ - {file = "types-setuptools-57.4.9.tar.gz", hash = "sha256:536ef74744f8e1e4be4fc719887f886e74e4cf3c792b4a06984320be4df450b5"}, - {file = "types_setuptools-57.4.9-py3-none-any.whl", hash = "sha256:948dc6863373750e2cd0b223a84f1fb608414cde5e55cf38ea657b93aeb411d2"}, + {file = "types-setuptools-57.4.10.tar.gz", hash = "sha256:9a13513679c640f6616e2d9ab50d431c99ca8ae9848a97243f887c80fd5cf294"}, + {file = "types_setuptools-57.4.10-py3-none-any.whl", hash = "sha256:ddc98da82c12e1208012d65276641a132d3aadc78ecfff68fd3e17d85933a3c1"}, ] types-toml = [ {file = "types-toml-0.10.4.tar.gz", hash = "sha256:9340e7c1587715581bb13905b3af30b79fe68afaccfca377665d5e63b694129a"}, @@ -652,8 +671,8 @@ typing-extensions = [ {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] virtualenv = [ - {file = "virtualenv-20.13.2-py2.py3-none-any.whl", hash = "sha256:e7b34c9474e6476ee208c43a4d9ac1510b041c68347eabfe9a9ea0c86aa0a46b"}, - {file = "virtualenv-20.13.2.tar.gz", hash = "sha256:01f5f80744d24a3743ce61858123488e91cb2dd1d3bdf92adaf1bba39ffdedf0"}, + {file = "virtualenv-20.13.3-py2.py3-none-any.whl", hash = "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021"}, + {file = "virtualenv-20.13.3.tar.gz", hash = "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134"}, ] zipp = [ {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, diff --git a/pyproject.toml b/pyproject.toml index dcbd0e97..7d2495ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.6" cyclonedx-python-lib = ">= 2.0.0, < 3.0.0" +pip-requirements-parser = "^31.2.0" [tool.poetry.dev-dependencies] autopep8 = "^1.6.0" diff --git a/tests/fixtures/requirements-local-and-remote-packages.txt b/tests/fixtures/requirements-local-and-remote-packages.txt new file mode 100644 index 00000000..8135b5c9 --- /dev/null +++ b/tests/fixtures/requirements-local-and-remote-packages.txt @@ -0,0 +1,8 @@ +./myproject/certifi # comment +./myproject/chardet +-e ./myproject/idna.whl +./myproject/requests --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 +./myproject/urllib3 +https://example.com/mypackage.whl + +-r requirements-nested.txt diff --git a/tests/fixtures/requirements-nested.txt b/tests/fixtures/requirements-nested.txt new file mode 100644 index 00000000..cf4d7c0a --- /dev/null +++ b/tests/fixtures/requirements-nested.txt @@ -0,0 +1 @@ +./downloads/numpy-1.9.2-cp34-none-win32.whl diff --git a/tests/fixtures/requirements-private-packages.txt b/tests/fixtures/requirements-private-packages.txt new file mode 100644 index 00000000..dca55ced --- /dev/null +++ b/tests/fixtures/requirements-private-packages.txt @@ -0,0 +1,3 @@ +--extra-index-url https://mypi.org/simple/ + +mypackage==1.2.3 diff --git a/tests/fixtures/requirements-with-urls.txt b/tests/fixtures/requirements-with-urls.txt new file mode 100644 index 00000000..d6df7180 --- /dev/null +++ b/tests/fixtures/requirements-with-urls.txt @@ -0,0 +1,4 @@ +git+https://github.com/path/to/package-one@41b95ec#egg=package-one # commit hash +git+https://github.com/path/to/package-two@master#egg=package-two # master branch +git+https://github.com/path/to/package-three@0.1#egg=package-three # tag +git+https://github.com/path/to/package-four@releases/tag/v3.7.1#egg=package-four # release tag diff --git a/tests/test_parser_requirements.py b/tests/test_parser_requirements.py index 3da08c88..26530b0d 100644 --- a/tests/test_parser_requirements.py +++ b/tests/test_parser_requirements.py @@ -18,66 +18,100 @@ # Copyright (c) OWASP Foundation. All Rights Reserved. import os -import unittest from unittest import TestCase -from cyclonedx_py.parser.requirements import RequirementsParser +from cyclonedx_py.parser.requirements import RequirementsFileParser, RequirementsParser class TestRequirementsParser(TestCase): def test_simple(self) -> None: - with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-simple.txt')) as r: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-simple.txt')) as r: parser = RequirementsParser( requirements_content=r.read() ) - r.close() self.assertTrue(1, parser.component_count()) self.assertFalse(parser.has_warnings()) def test_example_1(self) -> None: - with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-example-1.txt')) as r: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-example-1.txt')) as r: parser = RequirementsParser( requirements_content=r.read() ) - r.close() self.assertTrue(3, parser.component_count()) self.assertFalse(parser.has_warnings()) def test_example_with_comments(self) -> None: - with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-with-comments.txt')) as r: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-with-comments.txt')) as r: parser = RequirementsParser( requirements_content=r.read() ) - r.close() self.assertTrue(5, parser.component_count()) self.assertFalse(parser.has_warnings()) def test_example_multiline_with_comments(self) -> None: - with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-multilines-with-comments.txt')) as r: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-multilines-with-comments.txt')) as r: parser = RequirementsParser( requirements_content=r.read() ) - r.close() self.assertTrue(5, parser.component_count()) self.assertFalse(parser.has_warnings()) - @unittest.skip('Not yet supported') + def test_example_local_packages(self) -> None: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-local-and-remote-packages.txt')) as r: + parser = RequirementsParser( + requirements_content=r.read() + ) + self.assertTrue(6, parser.component_count()) + self.assertFalse(parser.has_warnings()) + + def test_example_local_and_nested_packages(self) -> None: + # RequirementsFileParser can parse nested requirements files, + # but RequirementsParser cannot. + parser = RequirementsFileParser( + requirements_file='fixtures/requirements-local-and-remote-packages.txt' + ) + self.assertTrue(7, parser.component_count()) + self.assertFalse(parser.has_warnings()) + + def test_example_private_packages(self) -> None: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-private-packages.txt')) as r: + parser = RequirementsParser( + requirements_content=r.read() + ) + self.assertTrue(1, parser.component_count()) + self.assertFalse(parser.has_warnings()) + + def test_example_with_urls(self) -> None: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-with-urls.txt')) as r: + parser = RequirementsParser( + requirements_content=r.read() + ) + self.assertTrue(4, parser.component_count()) + self.assertFalse(parser.has_warnings()) + def test_example_with_hashes(self) -> None: - with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-with-hashes.txt')) as r: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-with-hashes.txt')) as r: parser = RequirementsParser( requirements_content=r.read() ) - r.close() self.assertTrue(5, parser.component_count()) self.assertFalse(parser.has_warnings()) def test_example_without_pinned_versions(self) -> None: - with open(os.path.join(os.path.dirname(__file__), 'fixtures/requirements-without-pinned-versions.txt')) as r: + with open(os.path.join(os.path.dirname(__file__), + 'fixtures/requirements-without-pinned-versions.txt')) as r: parser = RequirementsParser( requirements_content=r.read() ) - r.close() self.assertTrue(2, parser.component_count()) self.assertTrue(parser.has_warnings()) self.assertEqual(3, len(parser.get_warnings()))