Add improver pipeline to flag ghost packages #644 #917 #1395

.github/workflows/docs.yml

@@ -9,7 +9,7 @@ jobs:
  strategy:
  max-parallel: 4
  matrix:
- python-version: [3.8]
+ python-version: [3.9]
 
  steps:
  - name: Checkout code

.github/workflows/main.yml

@@ -29,7 +29,7 @@ jobs:
  strategy:
  max-parallel: 4
  matrix:
- python-version: ["3.8", "3.9", "3.10"]
+ python-version: ["3.9", "3.10", "3.11"]
 
  steps:
  - name: Checkout code

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Release notes
 =============
 
+Version (next)
+-------------------
+
+- Add Pipeline to flag ghost packages (#1533)
+- Add logging configuration (#1533)
+- Drop support for python 3.8 (#1533)
+
+
 Version v34.0.0
 -------------------
 

requirements.txt

@@ -1,3 +1,4 @@
+aboutcode.pipeline==0.1.0
 aiosignal==1.2.0
 alabaster==0.7.12
 asgiref==3.5.2
@@ -10,6 +11,7 @@ bcrypt==3.2.0
 beautifulsoup4==4.10.0
 binaryornot==0.4.4
 black==22.3.0
+bleach==6.1.0
 boolean.py==3.8
 certifi==2024.7.4
 cffi==1.15.0
@@ -49,6 +51,7 @@ jsonschema==3.2.0
 license-expression==21.6.14
 lxml==4.9.1
 Markdown==3.3.4
+markdown-it-py==3.0.0
 MarkupSafe==2.1.1
 matplotlib-inline==0.1.3
 multidict==6.0.2

setup.cfg

@@ -48,7 +48,7 @@ license_files =
  README.rst
 
 [options]
-python_requires = >=3.8
+python_requires = >=3.9
 
 packages=find:
 include_package_data = true
@@ -92,6 +92,9 @@ install_requires =
  requests>=2.25.1
  fetchcode>=0.3.0
 
+ #pipeline
+ aboutcode.pipeline>=0.1.0
+
  #vulntotal
  python-dotenv
  texttable

vulnerabilities/improvers/__init__.py

@@ -10,6 +10,7 @@
 from vulnerabilities.improvers import valid_versions
 from vulnerabilities.improvers import vulnerability_kev
 from vulnerabilities.improvers import vulnerability_status
+from vulnerabilities.pipelines import flag_ghost_packages
 
 IMPROVERS_REGISTRY = [
  valid_versions.GitHubBasicImprover,
@@ -29,6 +30,7 @@
  valid_versions.GithubOSVImprover,
  vulnerability_status.VulnerabilityStatusImprover,
  vulnerability_kev.VulnerabilityKevImprover,
+ flag_ghost_packages.FlagGhostPackagePipeline,
 ]
 
 IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}

vulnerabilities/management/commands/improve.py

@@ -14,6 +14,7 @@
 
 from vulnerabilities.improve_runner import ImproveRunner
 from vulnerabilities.improvers import IMPROVERS_REGISTRY
+from vulnerabilities.pipelines import VulnerableCodePipeline
 
 
 class Command(BaseCommand):
@@ -56,6 +57,13 @@ def improve_data(self, improvers):
 
  for improver in improvers:
  self.stdout.write(f"Improving data using {improver.qualified_name}")
+ if issubclass(improver, VulnerableCodePipeline):
+ status, error = improver().execute()
+ if status != 0:
+ self.stdout.write(error)
+ failed_improvers.append(improver.qualified_name)
+ continue
+
  try:
  ImproveRunner(improver_class=improver).run()
  self.stdout.write(

vulnerabilities/migrations/0062_package_is_ghost.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.13 on 2024-08-23 12:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("vulnerabilities", "0061_alter_packagechangelog_software_version_and_more"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="package",
+ name="is_ghost",
+ field=models.BooleanField(
+ default=False,
+ help_text="True if the package does not exist in the upstream package manager or its repository.",
+ ),
+ ),
+ ]

vulnerabilities/models.py

@@ -610,6 +610,11 @@ class Package(PackageURLMixin):
  db_index=True,
  )
 
+ is_ghost = models.BooleanField(
+ default=False,
+ help_text="True if the package does not exist in the upstream package manager or its repository.",
+ )
+
  objects = PackageQuerySet.as_manager()
 
  def save(self, *args, **kwargs):

vulnerabilities/pipelines/__init__.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import logging
+from datetime import datetime
+from datetime import timezone
+
+from aboutcode.pipeline import BasePipeline
+
+from vulnerabilities.utils import classproperty
+
+module_logger = logging.getLogger(__name__)
+
+
+class VulnerableCodePipeline(BasePipeline):
+ def log(self, message, level=logging.INFO):
+ """Log the given `message` to the current module logger and execution_log."""
+ now_local = datetime.now(timezone.utc).astimezone()
+ timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+ message = f"{timestamp} {message}"
+ module_logger.log(level, message)
+ self.append_to_log(message)
+
+ @classproperty
+ def qualified_name(cls):
+ """
+ Fully qualified name prefixed with the module name of the pipeline used in logging.
+ """
+ return f"{cls.__module__}.{cls.__qualname__}"

vulnerabilities/pipelines/flag_ghost_packages.py

@@ -0,0 +1,102 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+from itertools import groupby
+from traceback import format_exc as traceback_format_exc
+
+from aboutcode.pipeline import LoopProgress
+from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS
+from fetchcode.package_versions import versions
+from packageurl import PackageURL
+
+from vulnerabilities.models import Package
+from vulnerabilities.pipelines import VulnerableCodePipeline
+
+
+class FlagGhostPackagePipeline(VulnerableCodePipeline):
+ """Detect and flag packages that do not exist upstream."""
+
+ @classmethod
+ def steps(cls):
+ return (cls.flag_ghost_packages,)
+
+ def flag_ghost_packages(self):
+ detect_and_flag_ghost_packages(logger=self.log)
+
+
+def detect_and_flag_ghost_packages(logger=None):
+ """Check if packages are available upstream. If not, mark them as ghost package."""
+ interesting_packages_qs = (
+ Package.objects.order_by("type", "namespace", "name")
+ .filter(type__in=FETCHCODE_SUPPORTED_ECOSYSTEMS)
+ .filter(qualifiers="")
+ .filter(subpath="")
+ )
+
+ distinct_packages_count = (
+ interesting_packages_qs.values("type", "namespace", "name")
+ .distinct("type", "namespace", "name")
+ .count()
+ )
+
+ grouped_packages = groupby(
+ interesting_packages_qs.paginated(),
+ key=lambda pkg: (pkg.type, pkg.namespace, pkg.name),
+ )
+
+ ghost_package_count = 0
+ progress = LoopProgress(total_iterations=distinct_packages_count, logger=logger)
+ for type_namespace_name, packages in progress.iter(grouped_packages):
+ ghost_package_count += flag_ghost_packages(
+ base_purl=PackageURL(*type_namespace_name),
+ packages=packages,
+ logger=logger,
+ )
+
+ if logger:
+ logger(f"Successfully flagged {ghost_package_count:,d} ghost Packages")
+
+
+def flag_ghost_packages(base_purl, packages, logger=None):
+ """
+ Check if `packages` are available upstream.
+ If not, update `is_ghost` to `True`.
+ Return the number of packages flagged as ghost.
+ """
+ known_versions = get_versions(purl=base_purl, logger=logger)
+ # Skip if encounter error while fetching known versions
+ if known_versions is None:
+ return 0
+
+ ghost_packages = 0
+ for pkg in packages:
+ pkg.is_ghost = False
+ if pkg.version.lstrip("vV") not in known_versions:
+ pkg.is_ghost = True
+ ghost_packages += 1
+
+ if logger:
+ logger(f"Flagging ghost package {pkg.purl!s}", level=logging.DEBUG)
+ pkg.save()
+
+ return ghost_packages
+
+
+def get_versions(purl, logger=None):
+ """Return set of known versions for the given purl."""
+ try:
+ return {v.value.lstrip("vV") for v in versions(str(purl))}
+ except Exception as e:
+ if logger:
+ logger(
+ f"Error while fetching known versions for {purl!s}: {e!r} \n {traceback_format_exc()}",
+ level=logging.ERROR,
+ )
+ return

vulnerabilities/templates/package_details.html

@@ -62,6 +62,21 @@
  {{ fixed_package_details.purl.to_string }}
  </td>
  </tr>
+ {% if package.is_ghost %}
+ <tr>
+ <td class="two-col-left">
+ Tags
+ </td>
+ <td class="two-col-right">
+ <span
+ class="tag is-warning is-hoverablem has-tooltip-multiline has-tooltip-black"
+ data-tooltip="This package does not exist in the upstream package manager or its repository."
+ style="margin-right: 8px;">
+ Ghost
+ </span>
+ </td>
+ </tr>
+ {% endif %}
  </tbody>
  </table>
  </div>

vulnerabilities/tests/pipelines/__init__.py

@@ -0,0 +1,20 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import io
+
+
+class TestLogger:
+ buffer = io.StringIO()
+
+ def write(self, msg, level=None):
+ self.buffer.write(msg)
+
+ def getvalue(self):
+ return self.buffer.getvalue()

vulnerabilities/tests/pipelines/test_flag_ghost_packages.py

@@ -0,0 +1,71 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+from pathlib import Path
+from unittest import mock
+
+from django.test import TestCase
+from fetchcode.package_versions import PackageVersion
+from packageurl import PackageURL
+
+from vulnerabilities.models import Package
+from vulnerabilities.pipelines import flag_ghost_packages
+from vulnerabilities.tests.pipelines import TestLogger
+
+
+class FlagGhostPackagePipelineTest(TestCase):
+ data = Path(__file__).parent.parent / "test_data"
+
+ @mock.patch("vulnerabilities.pipelines.flag_ghost_packages.versions")
+ def test_flag_ghost_package(self, mock_fetchcode_versions):
+ Package.objects.create(type="pypi", name="foo", version="2.3.0")
+ Package.objects.create(type="pypi", name="foo", version="3.0.0")
+
+ mock_fetchcode_versions.return_value = [
+ PackageVersion(value="2.3.0"),
+ ]
+ interesting_packages_qs = Package.objects.all()
+ base_purl = PackageURL(type="pypi", name="foo")
+
+ self.assertEqual(0, Package.objects.filter(is_ghost=True).count())
+
+ flagged_package_count = flag_ghost_packages.flag_ghost_packages(
+ base_purl=base_purl,
+ packages=interesting_packages_qs,
+ )
+ self.assertEqual(1, flagged_package_count)
+ self.assertEqual(1, Package.objects.filter(is_ghost=True).count())
+
+ @mock.patch("vulnerabilities.pipelines.flag_ghost_packages.versions")
+ def test_detect_and_flag_ghost_packages(self, mock_fetchcode_versions):
+ Package.objects.create(type="pypi", name="foo", version="2.3.0")
+ Package.objects.create(type="pypi", name="foo", version="3.0.0")
+ Package.objects.create(
+ type="deb",
+ namespace="debian",
+ name="foo",
+ version="3.0.0",
+ qualifiers={"distro": "trixie"},
+ )
+
+ mock_fetchcode_versions.return_value = [
+ PackageVersion(value="2.3.0"),
+ ]
+
+ self.assertEqual(3, Package.objects.count())
+ self.assertEqual(0, Package.objects.filter(is_ghost=True).count())
+
+ logger = TestLogger()
+
+ flag_ghost_packages.detect_and_flag_ghost_packages(logger=logger.write)
+ expected = "Successfully flagged 1 ghost Packages"
+
+ self.assertIn(expected, logger.getvalue())
+ self.assertEqual(1, Package.objects.filter(is_ghost=True).count())