diff --git a/scanpipe/models.py b/scanpipe/models.py index fd72fe711f..61c626e4ba 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -2587,7 +2587,7 @@ def compute_compliance_alert(self): return "" licensing = get_licensing() - parsed_symbols = licensing.parse(license_expression, simple=True).symbols + parsed_symbols = licensing.license_symbols(license_expression, simple=True) alerts = [ self.get_alert_for_symbol(license_policy_index, symbol) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 0a3c086063..767f69c9ba 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -158,7 +158,7 @@ def match_archives_to_purldb(self): d2d.match_purldb_resources( project=self.project, - extensions=self.matchable_package_extensions, + extensions=self.ecosystem_config.matchable_package_extensions, matcher_func=d2d.match_purldb_package, logger=self.log, ) @@ -249,7 +249,7 @@ def match_resources_to_purldb(self): d2d.match_purldb_resources( project=self.project, - extensions=self.matchable_resource_extensions, + extensions=self.ecosystem_config.matchable_resource_extensions, matcher_func=d2d.match_purldb_resource, logger=self.log, ) diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index 3a755f7426..1fdc028aa9 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -184,6 +184,5 @@ def add_ecosystem_config(pipeline_ecosystem_config, ecosystem_config): if not pipeline_config_value: new_config_value = config_value else: - new_config_value = pipeline_config_value.extend(config_value) - + new_config_value = config_value + pipeline_config_value setattr(pipeline_ecosystem_config, config_name, new_config_value) diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index 19d5b43876..ea38174917 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -333,6 +333,14 @@ def make_mock_response(url, content=b"\x00", status_code=200, headers=None): "label": "Prohibited License", "compliance_alert": "error", }, + { + "license_key": "gpl-2.0-plus", + "compliance_alert": "warning", + }, + { + "license_key": "font-exception-gpl", + "compliance_alert": "warning", + }, { "license_key": "OFL-1.1", "compliance_alert": "warning", @@ -368,6 +376,14 @@ def make_mock_response(url, content=b"\x00", status_code=200, headers=None): "label": "Prohibited License", "compliance_alert": "error", }, + "gpl-2.0-plus": { + "license_key": "gpl-2.0-plus", + "compliance_alert": "warning", + }, + "font-exception-gpl": { + "license_key": "font-exception-gpl", + "compliance_alert": "warning", + }, "OFL-1.1": { "license_key": "OFL-1.1", "compliance_alert": "warning", diff --git a/scanpipe/tests/data/d2d/config/ecosystem_config.json b/scanpipe/tests/data/d2d/config/ecosystem_config.json new file mode 100644 index 0000000000..10409ff2dc --- /dev/null +++ b/scanpipe/tests/data/d2d/config/ecosystem_config.json @@ -0,0 +1,10 @@ +{ + "ecosystem_option": "Default", + "matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"], + "matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"], + "doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"], + "deployed_resource_path_exclusions": ["*checksums.yaml.gz*", "*metadata.gz*"], + "devel_resource_path_exclusions": ["*/tests/*"], + "standard_symbols_to_exclude": [], + "source_symbol_extensions": [] +} \ No newline at end of file diff --git a/scanpipe/tests/data/policies/policies.yml b/scanpipe/tests/data/policies/policies.yml index 65f24e65d2..39706552f6 100644 --- a/scanpipe/tests/data/policies/policies.yml +++ b/scanpipe/tests/data/policies/policies.yml @@ -12,6 +12,13 @@ license_policies: label: Prohibited License compliance_alert: error + - license_key: gpl-2.0-plus + compliance_alert: warning + + # AboutCode license execption key + - license_key: font-exception-gpl + compliance_alert: warning + # SPDX license keys - license_key: OFL-1.1 compliance_alert: warning diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 1a808c5662..3ef0302126 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -21,9 +21,11 @@ # Visit https://github.com/nexB/scancode.io for support and download. import io +import json import sys import tempfile import uuid +from dataclasses import asdict from pathlib import Path from unittest import mock from unittest import skipIf @@ -1864,3 +1866,21 @@ def test_scanpipe_pipes_d2d_map_javascript_strings(self): map_type="javascript_strings", ).count(), ) + + def test_scanpipe_d2d_load_ecosystem_config(self): + pipeline_name = "map_deploy_to_develop" + selected_groups = ["Ruby", "Java", "JavaScript"] + + run = self.project1.add_pipeline( + pipeline_name=pipeline_name, selected_groups=selected_groups + ) + pipeline = run.make_pipeline_instance() + d2d_config.load_ecosystem_config(pipeline=pipeline, options=selected_groups) + + expected_ecosystem_config = ( + self.data / "d2d" / "config" / "ecosystem_config.json" + ) + with open(expected_ecosystem_config) as f: + expected_extra_data = json.load(f) + + self.assertEqual(expected_extra_data, asdict(pipeline.ecosystem_config)) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index a81af58bf2..78f7d189c9 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -1637,6 +1637,14 @@ def test_scanpipe_codebase_resource_model_compliance_alert_update_fields(self): resource.refresh_from_db() self.assertEqual("ok", resource.compliance_alert) + @patch.object(scanpipe_app, "policies", new=global_policies) + def test_scanpipe_can_compute_compliance_alert_for_license_exceptions(self): + scanpipe_app.license_policies_index = license_policies_index + resource = CodebaseResource.objects.create(project=self.project1, path="file") + license_expression = "gpl-2.0-plus WITH font-exception-gpl" + resource.update(detected_license_expression=license_expression) + self.assertEqual("warning", resource.compute_compliance_alert()) + def test_scanpipe_scan_fields_model_mixin_methods(self): expected = [ "detected_license_expression",