From 23d67240a35f6dbd121428ddf90a5d3209f2d40e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 4 Jul 2025 01:11:06 +0530 Subject: [PATCH 1/3] Fix d2d pipeline bugs from ecosytem configurations Reference: https://github.com/aboutcode-org/scancode.io/issues/1715 Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipelines/deploy_to_develop.py | 4 ++-- scanpipe/pipes/d2d_config.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 0a3c086063..767f69c9ba 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -158,7 +158,7 @@ def match_archives_to_purldb(self): d2d.match_purldb_resources( project=self.project, - extensions=self.matchable_package_extensions, + extensions=self.ecosystem_config.matchable_package_extensions, matcher_func=d2d.match_purldb_package, logger=self.log, ) @@ -249,7 +249,7 @@ def match_resources_to_purldb(self): d2d.match_purldb_resources( project=self.project, - extensions=self.matchable_resource_extensions, + extensions=self.ecosystem_config.matchable_resource_extensions, matcher_func=d2d.match_purldb_resource, logger=self.log, ) diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index 3a755f7426..ba396b2496 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -154,6 +154,9 @@ def load_ecosystem_config(pipeline, options): # Add default configurations which are common accross ecosystems pipeline.ecosystem_config = ECOSYSTEM_CONFIGS.get("Default") + if not options: + return + # Add configurations for each selected ecosystem for selected_option in options: if selected_option not in ECOSYSTEM_CONFIGS: @@ -184,6 +187,5 @@ def add_ecosystem_config(pipeline_ecosystem_config, ecosystem_config): if not pipeline_config_value: new_config_value = config_value else: - new_config_value = pipeline_config_value.extend(config_value) - + new_config_value = config_value + pipeline_config_value setattr(pipeline_ecosystem_config, config_name, new_config_value) From d6403f153d3d8b88e53ebcdb92167268d05575b1 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 7 Jul 2025 17:26:11 +0530 Subject: [PATCH 2/3] Fix license exception compliance parsing bug Reference: https://github.com/aboutcode-org/scancode.io/issues/1719 Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/models.py | 2 +- scanpipe/tests/__init__.py | 16 ++++++++++++++++ scanpipe/tests/data/policies/policies.yml | 7 +++++++ scanpipe/tests/test_models.py | 8 ++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index fe50d57b22..f1f3ac0723 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -2585,7 +2585,7 @@ def compute_compliance_alert(self): return "" licensing = get_licensing() - parsed_symbols = licensing.parse(license_expression, simple=True).symbols + parsed_symbols = licensing.license_symbols(license_expression, simple=True) alerts = [ self.get_alert_for_symbol(policy_index, symbol) for symbol in parsed_symbols diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index 19d5b43876..ea38174917 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -333,6 +333,14 @@ def make_mock_response(url, content=b"\x00", status_code=200, headers=None): "label": "Prohibited License", "compliance_alert": "error", }, + { + "license_key": "gpl-2.0-plus", + "compliance_alert": "warning", + }, + { + "license_key": "font-exception-gpl", + "compliance_alert": "warning", + }, { "license_key": "OFL-1.1", "compliance_alert": "warning", @@ -368,6 +376,14 @@ def make_mock_response(url, content=b"\x00", status_code=200, headers=None): "label": "Prohibited License", "compliance_alert": "error", }, + "gpl-2.0-plus": { + "license_key": "gpl-2.0-plus", + "compliance_alert": "warning", + }, + "font-exception-gpl": { + "license_key": "font-exception-gpl", + "compliance_alert": "warning", + }, "OFL-1.1": { "license_key": "OFL-1.1", "compliance_alert": "warning", diff --git a/scanpipe/tests/data/policies/policies.yml b/scanpipe/tests/data/policies/policies.yml index 65f24e65d2..39706552f6 100644 --- a/scanpipe/tests/data/policies/policies.yml +++ b/scanpipe/tests/data/policies/policies.yml @@ -12,6 +12,13 @@ license_policies: label: Prohibited License compliance_alert: error + - license_key: gpl-2.0-plus + compliance_alert: warning + + # AboutCode license execption key + - license_key: font-exception-gpl + compliance_alert: warning + # SPDX license keys - license_key: OFL-1.1 compliance_alert: warning diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index d0968f5235..a6d8cb6afc 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -1601,6 +1601,14 @@ def test_scanpipe_codebase_resource_model_compliance_alert_update_fields(self): # Reset the index value scanpipe_app.license_policies_index = None + def test_scanpipe_can_compute_compliance_alert_for_license_exceptions(self): + scanpipe_app.license_policies_index = license_policies_index + resource = CodebaseResource.objects.create(project=self.project1, path="file") + license_expression = "gpl-2.0-plus WITH font-exception-gpl" + resource.update(detected_license_expression=license_expression) + resource.compute_compliance_alert() + self.assertEqual("warning", resource.compliance_alert) + def test_scanpipe_scan_fields_model_mixin_methods(self): expected = [ "detected_license_expression", From ebacb9f51f90d0535d7d8961e1ef53d0f748d483 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 7 Jul 2025 21:31:23 +0530 Subject: [PATCH 3/3] Add tests for leading d2d configurations Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/d2d_config.py | 3 --- .../data/d2d/config/ecosystem_config.json | 10 ++++++++++ scanpipe/tests/pipes/test_d2d.py | 20 +++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 scanpipe/tests/data/d2d/config/ecosystem_config.json diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index ba396b2496..1fdc028aa9 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -154,9 +154,6 @@ def load_ecosystem_config(pipeline, options): # Add default configurations which are common accross ecosystems pipeline.ecosystem_config = ECOSYSTEM_CONFIGS.get("Default") - if not options: - return - # Add configurations for each selected ecosystem for selected_option in options: if selected_option not in ECOSYSTEM_CONFIGS: diff --git a/scanpipe/tests/data/d2d/config/ecosystem_config.json b/scanpipe/tests/data/d2d/config/ecosystem_config.json new file mode 100644 index 0000000000..10409ff2dc --- /dev/null +++ b/scanpipe/tests/data/d2d/config/ecosystem_config.json @@ -0,0 +1,10 @@ +{ + "ecosystem_option": "Default", + "matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"], + "matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"], + "doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"], + "deployed_resource_path_exclusions": ["*checksums.yaml.gz*", "*metadata.gz*"], + "devel_resource_path_exclusions": ["*/tests/*"], + "standard_symbols_to_exclude": [], + "source_symbol_extensions": [] +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 1a808c5662..3ef0302126 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -21,9 +21,11 @@ # Visit https://github.com/nexB/scancode.io for support and download. import io +import json import sys import tempfile import uuid +from dataclasses import asdict from pathlib import Path from unittest import mock from unittest import skipIf @@ -1864,3 +1866,21 @@ def test_scanpipe_pipes_d2d_map_javascript_strings(self): map_type="javascript_strings", ).count(), ) + + def test_scanpipe_d2d_load_ecosystem_config(self): + pipeline_name = "map_deploy_to_develop" + selected_groups = ["Ruby", "Java", "JavaScript"] + + run = self.project1.add_pipeline( + pipeline_name=pipeline_name, selected_groups=selected_groups + ) + pipeline = run.make_pipeline_instance() + d2d_config.load_ecosystem_config(pipeline=pipeline, options=selected_groups) + + expected_ecosystem_config = ( + self.data / "d2d" / "config" / "ecosystem_config.json" + ) + with open(expected_ecosystem_config) as f: + expected_extra_data = json.load(f) + + self.assertEqual(expected_extra_data, asdict(pipeline.ecosystem_config))