Merge pull request #3423 from nexB/fix-package-scan-only-performance

Fix package scan only performance

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -42,7 +42,17 @@ v32.1.0 (next, roadmap)
   See https://github.com/nexB/scancode-toolkit/issues/1745
 
 
-v32.0.3 - 2023-05-26
+v32.0.4 - 2023-06-07
+---------------------
+
+This is a minor bugfix release with the following updates:
+
+- Fixes a performance issue issue arising out of license detection
+  on files happening in a single-threaded process_codebase step when the
+  license CLI option is disabled for a package scan.
+  Reference: https://github.com/nexB/scancode-toolkit/pull/3423
+
+v32.0.3 - 2023-06-06
 ---------------------
 
 This is a minor bugfix release with the following updates:

setup-mini.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit-mini
-version = 32.0.3
+version = 32.0.4
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit
-version = 32.0.3
+version = 32.0.4
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

src/packagedcode/build.py

@@ -14,6 +14,8 @@
 
 from commoncode import fileutils
 
+from licensedcode.cache import build_spdx_license_expression
+from licensedcode.cache import get_cache
 from licensedcode.tokenize import query_tokenizer
 from licensedcode.detection import detect_licenses
 from licensedcode.detection import get_unknown_license_detection
@@ -122,6 +124,11 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                     resource=resource,
                     codebase=codebase,
                 )
+            if package.declared_license_expression:
+                package.declared_license_expression_spdx = str(build_spdx_license_expression(
+                    license_expression=package.declared_license_expression,
+                    licensing=get_cache().licensing,
+                ))
 
             cls.assign_package_to_resources(
                 package=package,
@@ -132,6 +139,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
             yield package
 
+
         # we yield this as we do not want this further processed
         yield resource
 

src/packagedcode/licensing.py

@@ -56,7 +56,7 @@ def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
-def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
+def add_referenced_license_matches_for_package(resource, codebase):
     """
     Return an updated ``resource`` saving it in place, after adding new license
     detections to the package manifests detected in this resource, following their
@@ -106,13 +106,7 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
                 if not referenced_resource:
                     continue
 
-                if no_licenses:
-                    referenced_license_detections = get_license_detection_mappings(
-                        location=referenced_resource.location
-                    )
-
-                else:
-                    referenced_license_detections = referenced_resource.license_detections
+                referenced_license_detections = referenced_resource.license_detections
 
                 if referenced_license_detections:
                     modified = True
@@ -160,7 +154,7 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
             yield resource
 
 
-def add_referenced_license_detection_from_package(resource, codebase, no_licenses):
+def add_referenced_license_detection_from_package(resource, codebase):
     """
     Return an updated ``resource`` saving it in place, after adding new license
     matches (licenses and license_expressions) following their Rule
@@ -209,7 +203,6 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
             sibling_license_detections, _le = get_license_detections_from_sibling_file(
                 resource=root_resource,
                 codebase=codebase,
-                no_licenses=no_licenses,
             )
             if TRACE:
                 logger_debug(
@@ -278,12 +271,10 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
         yield resource
 
 
-def add_license_from_sibling_file(resource, codebase, no_licenses):
+def add_license_from_sibling_file(resource, codebase):
     """
     Given a resource and it's codebase object, assign licenses to the package
     detections in that resource, from the sibling files of it.
-
-    If `no_license` is True, then license scan (for resources) is disabled.
     """
     if TRACE:
         logger_debug(f'packagedcode.licensing: add_license_from_sibling_file: resource: {resource.path}')
@@ -303,7 +294,6 @@ def add_license_from_sibling_file(resource, codebase, no_licenses):
     license_detections, license_expression = get_license_detections_from_sibling_file(
         resource=resource,
         codebase=codebase,
-        no_licenses=no_licenses,
     )
     if not license_detections:
         return
@@ -333,13 +323,11 @@ def is_legal_or_readme(resource):
     return False
 
 
-def get_license_detections_from_sibling_file(resource, codebase, no_licenses):
+def get_license_detections_from_sibling_file(resource, codebase):
     """
     Return `license_detections`, a list of LicenseDetection objects and a
     `license_expression`, given a resource and it's codebase object, from
     the sibling files of the resource.
-
-    If `no_license` is True, then license scan (for resources) is disabled.
     """
     siblings = []
 
@@ -357,15 +345,7 @@ def get_license_detections_from_sibling_file(resource, codebase, no_licenses):
 
     license_detections = []
     for sibling in siblings:
-        if no_licenses:
-            detections = get_license_detection_mappings(
-                location=sibling.location,
-                analysis=DetectionCategory.PACKAGE_ADD_FROM_SIBLING_FILE.value,
-                post_scan=True,
-            )
-            license_detections.extend(detections)
-        else:
-            license_detections.extend(sibling.license_detections)
+        license_detections.extend(sibling.license_detections)
 
     if not license_detections:
         return [], None

src/packagedcode/plugin_package.py

@@ -194,17 +194,19 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
         Also perform additional package license detection that depends on either
         file license detection or the package detections.
         """
-        no_licenses = False
+        has_licenses = hasattr(codebase.root, 'license_detections')
 
         # These steps add proper license detections to package_data and hence
         # this is performed before top level packages creation
         for resource in codebase.walk(topdown=False):
-            if not hasattr(resource, 'license_detections'):
-                no_licenses = True
+            if not has_licenses:
+                #TODO: Add the steps where we detect licenses from files for only a package scan
+                # in the multiprocessing get_package_data API function
+                continue
 
             # If we don't detect license in package_data but there is license detected in file
             # we add the license expression from the file to a package
-            modified = add_license_from_file(resource, codebase, no_licenses)
+            modified = add_license_from_file(resource, codebase)
             if TRACE and modified:
                 logger_debug(f'packagedcode: process_codebase: add_license_from_file: modified: {modified}')
 
@@ -213,30 +215,30 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
 
             # If there is referenced files in a extracted license statement, we follow
             # the references, look for license detections and add them back
-            modified = list(add_referenced_license_matches_for_package(resource, codebase, no_licenses))
+            modified = list(add_referenced_license_matches_for_package(resource, codebase))
             if TRACE and modified:
                 logger_debug(f'packagedcode: process_codebase: add_referenced_license_matches_for_package: modified: {modified}')
 
             # If there is a LICENSE file on the same level as the manifest, and no license
             # is detected in the package_data, we add the license from the file
-            modified = add_license_from_sibling_file(resource, codebase, no_licenses)
+            modified = add_license_from_sibling_file(resource, codebase)
             if TRACE and modified:
                 logger_debug(f'packagedcode: process_codebase: add_license_from_sibling_file: modified: {modified}')
 
         # Create codebase-level packages and dependencies
         create_package_and_deps(codebase, strip_root=strip_root, **kwargs)
 
-        if not no_licenses:
+        if has_licenses:
             # This step is dependent on top level packages
             for resource in codebase.walk(topdown=False):
                 # If there is a unknown reference to a package we add the license
                 # from the package license detection
-                modified = list(add_referenced_license_detection_from_package(resource, codebase, no_licenses))
+                modified = list(add_referenced_license_detection_from_package(resource, codebase))
                 if TRACE and modified:
                     logger_debug(f'packagedcode: process_codebase: add_referenced_license_matches_from_package: modified: {modified}')
 
 
-def add_license_from_file(resource, codebase, no_licenses):
+def add_license_from_file(resource, codebase):
     """
     Given a Resource, check if the detected package_data doesn't have license detections
     and the file has license detections, and if so, populate the package_data license
@@ -248,10 +250,7 @@ def add_license_from_file(resource, codebase, no_licenses):
     if not resource.is_file:
         return
 
-    if no_licenses:
-        license_detections_file = get_license_detection_mappings(location=resource.location)
-    else:
-        license_detections_file = resource.license_detections
+    license_detections_file = resource.license_detections
 
     if TRACE:
         logger_debug(f'add_license_from_file: license_detections_file: {license_detections_file}')

src/scancode_config.py

@@ -132,11 +132,12 @@ def _create_dir(location):
 # 4. hardcoded This is the default, fallback version in case package is not installed or we
 # do not have a proper version otherwise.
 if not __version__:
-    __version__ = '32.0.3'
+    __version__ = '32.0.4'
 
 #######################
 # used to warn user when the version is out of date
-__release_date__ = datetime.datetime(2023, 6, 6)
+# this is (year, month, day)
+__release_date__ = datetime.datetime(2023, 6, 7)
 
 # See https://github.com/nexB/scancode-toolkit/issues/2653 for more information
 # on the data format version

tests/formattedcode/test_output_cyclonedx.py

@@ -228,22 +228,22 @@ def test_cyclonedx_plugin_does_not_fail_without_packages():
 def test_cyclonedx_plugin_json():
     test_dir = test_env.get_test_loc('cyclonedx/simple')
     result_file = test_env.get_temp_file('cyclonedx.json')
-    run_scan_click(['-p', test_dir, '--cyclonedx', result_file])
+    run_scan_click(['--package', test_dir, '--cyclonedx', result_file])
     expected_file = test_env.get_test_loc('cyclonedx/simple-expected.json')
     check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_cyclonedx_plugin_json_simple_package_icu():
     test_dir = test_env.get_test_loc('cyclonedx/simple-icu')
     result_file = test_env.get_temp_file('cyclonedx.json')
-    run_scan_click(['-p', test_dir, '--cyclonedx', result_file])
+    run_scan_click(['--package', '--license', test_dir, '--cyclonedx', result_file])
     expected_file = test_env.get_test_loc('cyclonedx/simple-icu-expected.json')
     check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_cyclonedx_plugin_xml_components_and_dependencies_are_serialized_correctly():
     test_dir = test_env.get_test_loc('cyclonedx/simple')
     result_file = test_env.get_temp_file('cyclonedx.xml')
-    run_scan_click(['-p', test_dir, '--cyclonedx-xml', result_file])
+    run_scan_click(['--package', test_dir, '--cyclonedx-xml', result_file])
     expected_file = test_env.get_test_loc('cyclonedx/expected.xml')
     check_cyclone_xml_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES)

tests/packagedcode/data/build/buck/end2end-expected.json

@@ -247,29 +247,9 @@
           "vcs_url": null,
           "copyright": null,
           "holder": null,
-          "declared_license_expression": "apache-2.0",
-          "declared_license_expression_spdx": "Apache-2.0",
-          "license_detections": [
-            {
-              "license_expression": "apache-2.0",
-              "matches": [
-                {
-                  "score": 100.0,
-                  "start_line": 1,
-                  "end_line": 1,
-                  "matched_length": 3,
-                  "match_coverage": 100.0,
-                  "matcher": "1-hash",
-                  "license_expression": "apache-2.0",
-                  "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE",
-                  "rule_relevance": 100,
-                  "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE",
-                  "matched_text": "apache-2.0"
-                }
-              ],
-              "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8"
-            }
-          ],
+          "declared_license_expression": null,
+          "declared_license_expression_spdx": null,
+          "license_detections": [],
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],