diff --git a/minecode/collectors/bitbucket.py b/minecode/collectors/bitbucket.py new file mode 100644 index 00000000..11c60f1b --- /dev/null +++ b/minecode/collectors/bitbucket.py @@ -0,0 +1,50 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging + +import requests + + +""" +Collect bitbucket packages from bitbucket registries. +""" + +logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +def bitbucket_get_all_package_version_author(subset_path): + """ + Return a list of all version numbers along with author for the package. + """ + repo_tags = f"https://api.bitbucket.org/2.0/repositories/{subset_path}/refs/tags" + version_author_list = [] + try: + while repo_tags: + response = requests.get(repo_tags) + response.raise_for_status() + data = response.json() + if data["size"] > 0: + # Get all available versions + for item in data["values"]: + version = item.get("name") + target = item.get("target") or {} + author = target.get("author") or {} + if author.get("type") == "author": + user = author.get("user") or {} + author_display_name = user.get("display_name") + version_author_list.append((version, author_display_name)) + # Handle pagination + repo_tags = data.get("next", None) + return version_author_list + except requests.exceptions.HTTPError as err: + logger.error(f"HTTP error occurred: {err}") diff --git a/minecode/collectors/generic.py b/minecode/collectors/generic.py index 032be878..09cbf395 100644 --- a/minecode/collectors/generic.py +++ b/minecode/collectors/generic.py @@ -100,7 +100,7 @@ def packagedata_from_dict(package_data): return PackageData.from_data(cleaned_package_data) -def map_fetchcode_supported_package(package_url, pipelines, priority=0): +def map_fetchcode_supported_package(package_url, pipelines, priority=0, from_go_lang=False): """ Add a `package_url` supported by fetchcode to the PackageDB. @@ -109,13 +109,24 @@ def map_fetchcode_supported_package(package_url, pipelines, priority=0): from minecode.model_utils import add_package_to_scan_queue from minecode.model_utils import merge_or_create_package - packages = [p for p in info(str(package_url)) or []] + try: + packages = [] + packages = [p for p in info(str(package_url)) or []] + except Exception as e: + print(str(e)) if not packages: - error = f"Could not find package using fetchcode: {package_url}" + if from_go_lang: + purl = "pkg:golang/" + str(package_url).partition("pkg:")[2] + else: + purl = str(package_url) + error = f"Could not find package using fetchcode: {purl}" logger.error(error) return error + if from_go_lang: + packages[0].type = "golang" + packages[0].namespace = "github.com/" + packages[0].namespace package_data = packages[0].to_dict() # Remove obsolete Package fields see https://github.com/aboutcode-org/fetchcode/issues/108 diff --git a/minecode/collectors/github.py b/minecode/collectors/github.py index fbf6337e..3151faf0 100644 --- a/minecode/collectors/github.py +++ b/minecode/collectors/github.py @@ -13,6 +13,42 @@ from minecode.collectors.generic import map_fetchcode_supported_package +def github_get_all_versions(subset_path): + """ + Fetch all versions (tags) from a GitHub repository using the API + Returns a list of all version tags in the repository + """ + import requests + + url = f"https://api.github.com/repos/{subset_path}/tags" + version_list = [] + page = 1 + + while True: + response = requests.get( + url, + params={"page": page, "per_page": 100}, # Max 100 per page + headers={"Accept": "application/vnd.github.v3+json"}, + ) + response.raise_for_status() + + data = response.json() + if not data: + break + + for tag in data: + version = tag.get("name") or {} + if version: + version_list.append(version) + page += 1 + + # Check if we've reached the last page + if "next" not in response.links: + break + + return version_list + + # Indexing GitHub PURLs requires a GitHub API token. # Please add your GitHub API key to the `.env` file, for example: `GH_TOKEN=your-github-api`. @priority_router.route("pkg:github/.*") diff --git a/minecode/collectors/gitlab.py b/minecode/collectors/gitlab.py new file mode 100644 index 00000000..3f98b918 --- /dev/null +++ b/minecode/collectors/gitlab.py @@ -0,0 +1,45 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging + +import requests + + +""" +Collect gitlab packages from gitlab registries. +""" + +logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +def gitlab_get_all_package_version_author(subset_path): + """ + Return a list of all version numbers along with author and author email + for the package. + """ + repo_tags = f"https://gitlab.com/api/v4/projects/{subset_path}/repository/tags" + try: + response = requests.get(repo_tags) + response.raise_for_status() + data = response.json() + version_author_list = [] + # Get all available versions + for item in data: + version = item.get("name") + commit = item.get("commit") or {} + author = commit.get("author_name") or "" + author_email = commit.get("author_email") or "" + version_author_list.append((version, author, author_email)) + return version_author_list + except requests.exceptions.HTTPError as err: + logger.error(f"HTTP error occurred: {err}") diff --git a/minecode/collectors/golang.py b/minecode/collectors/golang.py new file mode 100644 index 00000000..34de52b6 --- /dev/null +++ b/minecode/collectors/golang.py @@ -0,0 +1,370 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import logging +import requests + +from minecode import priority_router +from minecode.collectors.generic import map_fetchcode_supported_package +from minecode.collectors.gitlab import gitlab_get_all_package_version_author +from minecode.collectors.github import github_get_all_versions +from minecode.collectors.bitbucket import bitbucket_get_all_package_version_author +from minecode.miners.gitlab import build_packages_from_json_golang +from minecode.miners.golang import build_golang_generic_package +from minecode.miners.bitbucket import build_bitbucket_packages + +from packageurl import PackageURL + +from packagedb.models import PackageContentType + +from urllib.parse import quote + +logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +def extract_golang_subset_purl(purl_str): + """ + Extract the first two segments after github.com or bitbucket.org and + version For instance, pkg:golang/github.com/rickar/cal/v2/aa@2.1.23 + Return + subset_path: rickar/cal + version: 2.1.23 + """ + # Strip "pkg:golang/" + purl_body = purl_str[len("pkg:golang/") :] + + # Extract namespace, name, and version + parts = purl_body.split("/") + version = "" + if "@" in purl_str: + version = purl_str.rpartition("@")[2] + subset_path = parts[1] + "/" + parts[2].partition("@")[0] + + return subset_path, version + + +def gitlab_updated_purl(purl_str): + """ + Return the path between "pkg:golang/gitlab.com/" and version with + replacing "/" with "%2F" and version + """ + version = "" + if "@" in purl_str: + version = purl_str.rpartition("@")[2] + subset = purl_str.partition("pkg:golang/gitlab.com/")[2].partition("@")[0] + subset_path = quote(subset, safe="") + return subset_path, version + + +def get_package_versions(namespace_name): + """ + Return all the version of the package fetched from "api.deps.dev" in a + list + """ + version_list = [] + url = f"https://api.deps.dev/v3/systems/GO/packages/{namespace_name}" + try: + response = requests.get(url) + response.raise_for_status() + context = response.json() + versions = context["versions"] + for version in versions: + version_list.append(version["versionKey"]["version"]) + return version_list + except requests.exceptions.HTTPError as err: + logger.error(f"HTTP error occurred: {err}") + + +def get_package_json(subset_path, type=None, version=None): + """ + Return the contents of the JSON file of the package. + """ + # Create URLs using purl fields + if type == "gitlab": + url = f"https://gitlab.com/api/v4/projects/{subset_path}" + elif type == "bitbucket": + url = f"https://api.bitbucket.org/2.0/repositories/{subset_path}" + else: + if version.startswith("v"): + url = f"https://api.deps.dev/v3/systems/GO/packages/{subset_path}/versions/{version}" + else: + url = f"https://api.deps.dev/v3/systems/GO/packages/{subset_path}/versions/v{version}" + try: + response = requests.get(url) + response.raise_for_status() + return response.json() + except requests.exceptions.HTTPError as err: + logger.error(f"HTTP error occurred: {err}") + except AttributeError: + logger.error("The PURL is missing a version.") + + +def map_golang_package(package_url, package_json, pipelines, priority=0, filename=None): + """ + Add a golang `package_url` to the PackageDB. + + Return an error string if any errors are encountered during the process + """ + from minecode.model_utils import add_package_to_scan_queue + from minecode.model_utils import merge_or_create_package + + error = "" + + if not package_json: + error = f"Package does not exist: {package_url}" + logger.error(error) + return error + + purl_str = package_url.to_string() + if purl_str.startswith("pkg:golang/gitlab"): + packages = build_packages_from_json_golang(package_json, package_url) + elif purl_str.startswith("pkg:golang/bitbucket"): + packages = build_bitbucket_packages(package_json, package_url) + else: + packages = build_golang_generic_package(package_json, package_url) + + for package in packages: + package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE + db_package, _, _, error = merge_or_create_package(package, visit_level=0, filename=filename) + if error: + break + + # Submit package for scanning + if db_package: + add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority) + + return error + + +def process_download_metadata(download_url, package_json): + """ + Return the download_url and the filename + """ + response = requests.head(download_url, allow_redirects=True) + redirected_download_url = response.url + # Sometimes, the filename obtained from a + # downloaded URL, even after following a redirect, + # does not match the actual name of the downloaded + # file. To retrieve the correct filename, it is + # necessary to examine the "Content-Disposition" + # header. + content_disposition = response.headers.get("Content-Disposition") + if content_disposition: + filename = content_disposition.split("filename=")[-1].strip('"') + else: + filename = redirected_download_url.rpartition("/")[2] + package_json["download_url"] = redirected_download_url + + return package_json, filename + + +# It may need indexing GitHub PURLs that requires a GitHub API token. +# Please add your GitHub API key to the `.env` file, for example: `GH_TOKEN=your-github-api`. +@priority_router.route("pkg:golang/.*") +def process_requests(purl_str, **kwargs): + """ + Process `priority_resource_uri` containing a golang Package URL (PURL). + + This involves obtaining Package information for the PURL using + https://github.com/aboutcode-org/fetchcode and using it to create a new + PackageDB entry. The package is then added to the scan queue afterwards. + """ + from minecode.model_utils import DEFAULT_PIPELINES + + addon_pipelines = kwargs.get("addon_pipelines", []) + pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines) + priority = kwargs.get("priority", 0) + + try: + """ + We retrieve metadata from APIs for GitHub, GitLab, and Bitbucket. + For the other cases (or failing cases), we will get the data from + deps.dev + """ + processed = False + if purl_str.startswith("pkg:golang/github"): + subset_path, version = extract_golang_subset_purl(purl_str) + if version: + # Construct the GitHub purl + github_purl = f"pkg:github/{subset_path}@{version}" + package_url = PackageURL.from_string(github_purl) + error_msg = map_fetchcode_supported_package( + package_url, pipelines, priority, from_go_lang=True + ) + if error_msg: + print(error_msg) + else: + processed = True + else: + version_list = github_get_all_versions(subset_path) + for v in version_list: + # Construct the GitHub purl + # Strip the 'version' or 'v' from the collected version + if v.startswith("version"): + v = v.partition("version")[2] + elif v.startswith("v"): + v = v[1:] + github_purl = f"pkg:github/{subset_path}@{v}" + package_url = PackageURL.from_string(github_purl) + error_msg = map_fetchcode_supported_package( + package_url, pipelines, priority, from_go_lang=True + ) + if error_msg: + print(error_msg) + else: + processed = True + elif purl_str.startswith("pkg:golang/gitlab"): + package_url = PackageURL.from_string(purl_str) + subset_path, version = gitlab_updated_purl(purl_str) + package_json = get_package_json(subset_path, "gitlab") + if not package_json: + error = f"package not found: {purl_str}" + else: + repo_version_author_list = gitlab_get_all_package_version_author(subset_path) + if repo_version_author_list: + for repo_version, author, email in repo_version_author_list: + # Check the version along with stripping the first + # character 'v' in the repo_version + if not version or version in {repo_version, repo_version[1:]}: + download_url = f"https://gitlab.com/api/v4/projects/{subset_path}/repository/archive.zip?sha={repo_version}" + updated_json, filename = process_download_metadata( + download_url, package_json + ) + updated_json["author"] = author + updated_json["email"] = email + if not version: + if repo_version.startswith("v"): + updated_purl_str = ( + PackageURL.to_string(package_url) + "@" + repo_version[1:] + ) + else: + updated_purl_str = ( + PackageURL.to_string(package_url) + "@" + repo_version + ) + updated_purl = PackageURL.from_string(updated_purl_str) + error_msg = map_golang_package( + updated_purl, + updated_json, + pipelines, + priority, + filename=filename, + ) + if error_msg: + print(error_msg) + else: + processed = True + else: + error_msg = map_golang_package( + package_url, + updated_json, + pipelines, + priority, + filename=filename, + ) + if error_msg: + print(error_msg) + else: + processed = True + break + else: + # The repo does not have any tag (i.e. it only has one version) + download_url = ( + f"https://gitlab.com/api/v4/projects/{subset_path}/repository/archive.zip" + ) + updated_json, filename = process_download_metadata(download_url, package_json) + error_msg = map_golang_package( + package_url, updated_json, pipelines, priority, filename=filename + ) + if error_msg: + print(error_msg) + else: + processed = True + elif purl_str.startswith("pkg:golang/bitbucket"): + package_url = PackageURL.from_string(purl_str) + subset_path, version = extract_golang_subset_purl(purl_str) + package_json = get_package_json(subset_path, "bitbucket") + if not package_json: + error = f"package not found: {purl_str}" + else: + repo_version_author_list = bitbucket_get_all_package_version_author(subset_path) + package_json["repo_workspace_name"] = subset_path + if repo_version_author_list: + found_match = False + for repo_version, author in repo_version_author_list: + # Check the version along with stripping the first + # character 'v' in the repo_version + if not version or version in {repo_version, repo_version[1:]}: + found_match = True + download_url = ( + f"https://bitbucket.org/{subset_path}/get/{repo_version}.zip" + ) + updated_json, filename = process_download_metadata( + download_url, package_json + ) + updated_json["author"] = author + if not version: + if repo_version.startswith("v"): + collected_version = repo_version[1:] + else: + collected_version = repo_version + updated_purl_str = purl_str + "@" + collected_version + package_url = PackageURL.from_string(updated_purl_str) + error_msg = map_golang_package( + package_url, updated_json, pipelines, priority, filename=filename + ) + if error_msg: + print(error_msg) + if version: + break + if not found_match: + error_msg = f"The package version not found: {version}" + else: + processed = True + else: + # The repo does not have any tag (i.e. it only has one version) + # Get the main branch name for the download url + main_branch = package_json["mainbranch"]["name"] + download_url = f"https://bitbucket.org/{subset_path}/get/{main_branch}.zip" + updated_json, filename = process_download_metadata(download_url, package_json) + + error_msg = map_golang_package( + package_url, updated_json, pipelines, priority, filename=filename + ) + if error_msg: + print(error_msg) + else: + processed = True + if not processed: + # Handle case which no version is in the input purl + if "@" not in purl_str: + namespace_name = purl_str.partition("pkg:golang/")[2] + encoded_namespace_name = quote(namespace_name, safe="") + version_list = get_package_versions(encoded_namespace_name) + + for version in version_list: + purl_str_version = purl_str + "@" + version + package_url = PackageURL.from_string(purl_str_version) + package_json = get_package_json(encoded_namespace_name, version=version) + error_msg = map_golang_package(package_url, package_json, pipelines, priority) + if error_msg: + print(error_msg) + else: + namespace_name = purl_str.partition("pkg:golang/")[2].rpartition("@")[0] + encoded_namespace_name = quote(namespace_name, safe="") + package_url = PackageURL.from_string(purl_str) + package_json = get_package_json(encoded_namespace_name, version=package_url.version) + error_msg = map_golang_package(package_url, package_json, pipelines, priority) + if error_msg: + print(error_msg) + + except ValueError as e: + error = f"error occurred when parsing {purl_str}: {e}" + return error diff --git a/minecode/miners/bitbucket.py b/minecode/miners/bitbucket.py index be3f968f..f9afe48b 100644 --- a/minecode/miners/bitbucket.py +++ b/minecode/miners/bitbucket.py @@ -4,6 +4,7 @@ import json import logging +import requests from packagedcode import models as scan_models from packageurl import PackageURL @@ -300,3 +301,88 @@ def build_bitbucket_repo_package(repo_data, purl): ) package.set_purl(purl) return package + + +def get_bitbucket_license_info(repo_path): + """ + Fetch license information from a Bitbucket repository. + Returns the detected license text based on the common license filenames + """ + + # Bitbucket API endpoint for repository sources (where license file typically is) + url = f"https://api.bitbucket.org/2.0/repositories/{repo_path}/src" + + try: + while url: + response = requests.get(url) + response.raise_for_status() + + # Check common license file names + common_license_file_name = [ + "LICENSE", + "LICENSE.MD", + "LICENSE.TXT", + "COPYING", + "COPYING.TXT", + ] + data = response.json() + # Search for license files in the root directory + for item in data.get("values"): + path = item.get("path") + if path.upper() in common_license_file_name: + # Found a license file - fetch its content + license_url = ( + f"https://api.bitbucket.org/2.0/repositories/{repo_path}/src/HEAD/{path}" + ) + license_response = requests.get(license_url) + license_response.raise_for_status() + return license_response.text + # Handle pagination + url = data.get("next", None) + return None # No license file found + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching license info: {e}") + return None + + +def build_bitbucket_packages(metadata_dict, purl): + """ + Yield ScannedPackage built from Bitbucket. + The metadata_dict is a dictionary. + purl: String value of the package url of the ResourceURI object + """ + name = metadata_dict.get("name") + description = metadata_dict.get("description") or "" + links = metadata_dict.get("links") or {} + html = links.get("html") or {} + homepage_url = html.get("href") or "" + size = metadata_dict.get("size") or "" + primary_language = metadata_dict.get("language") or "" + + if "repo_workspace_name" in metadata_dict: + repo_path = metadata_dict.get("repo_workspace_name") or "" + else: + repo_path = "" + license_text = get_bitbucket_license_info(repo_path) + extracted_license_statement = [license_text] + + common_data = dict( + name=name, + description=description, + homepage_url=homepage_url, + size=size, + primary_language=primary_language, + extracted_license_statement=extracted_license_statement, + ) + + download_data = dict( + datasource_id="bitbucket_pkginfo", + download_url=metadata_dict.get("download_url") or "", + ) + + common_data.update(download_data) + package = scan_models.PackageData.from_data(common_data) + + package.datasource_id = "bitbucket_api_metadata" + package.set_purl(purl) + yield package diff --git a/minecode/miners/gitlab.py b/minecode/miners/gitlab.py index 8a27f98c..f26662df 100644 --- a/minecode/miners/gitlab.py +++ b/minecode/miners/gitlab.py @@ -8,6 +8,7 @@ # import json +import requests import packagedcode.models as scan_models from packageurl import PackageURL @@ -134,3 +135,59 @@ def build_packages_from_json(metadata, purl=None): package = scan_models.Package(**common_data) package.set_purl(purl) yield package + + +def build_packages_from_json_golang(content, purl=None): + """ + Yield Package built from gitlab json content + metadata: Json metadata content + purl: String value of the package url of the ResourceURI object + """ + id = content.get("id") + name = content.get("name") + repository_homepage_url = content.get("http_url_to_repo") + version = "" + if purl: + version = purl.version + + author = "" + email = "" + if "author" in content: + author = content.get("author") + if "email" in content: + email = content.get("email") + + license_url = f"https://gitlab.com/api/v4/projects/{id}/repository/files/LICENSE/raw" + response = requests.get(license_url) + extracted_license_statement = [] + if response.status_code == 200: + extracted_license_statement = [response.text] + + common_data = dict( + name=name, + version=version, + description=content.get("description"), + homepage_url=content.get("web_url"), + repository_homepage_url=repository_homepage_url, + extracted_license_statement=extracted_license_statement, + download_url=content.get("download_url"), + primary_language="go", + ) + + if repository_homepage_url: + repository_homepage_url = form_vcs_url("git", repository_homepage_url) + common_data["vcs_url"] = repository_homepage_url + common_data["code_view_url"] = repository_homepage_url + common_data["release_date"] = parse_date(content.get("created_at")) + + if author: + parties = common_data.get("parties") + if not parties: + common_data["parties"] = [] + common_data["parties"].append(scan_models.Party(name=author, role="author", email=email)) + + package = scan_models.PackageData.from_data(common_data) + + package.datasource_id = "golang_api_metadata" + package.set_purl(purl) + yield package diff --git a/minecode/miners/golang.py b/minecode/miners/golang.py index e54593b6..6ea252ee 100644 --- a/minecode/miners/golang.py +++ b/minecode/miners/golang.py @@ -11,6 +11,7 @@ from packagedcode import models as scan_models from packageurl import PackageURL +from packageurl.contrib.purl2url import build_golang_download_url from minecode import map_router from minecode import seed @@ -237,3 +238,27 @@ def build_golang_package(package_data, purl): vcs_url=vcs_url, ) return package + + +def build_golang_generic_package(package_data, package_url): + """Return a single Golang package""" + homepage_url = "/".join(["https:/", package_url.namespace, package_url.name]) + license_text = package_data.get("licenses") + extracted_license_statement = [license_text] + + purl_str = package_url.to_string() + download_url = build_golang_download_url(purl_str) + + common_data = dict( + name=package_url.name, + namespace=package_url.namespace, + type=package_url.type, + primary_language="go", + homepage_url=homepage_url, + extracted_license_statement=extracted_license_statement, + download_url=download_url, + ) + + package = scan_models.PackageData.from_data(common_data) + package.set_purl(package_url) + yield package diff --git a/minecode/model_utils.py b/minecode/model_utils.py index f31245ab..50f04800 100644 --- a/minecode/model_utils.py +++ b/minecode/model_utils.py @@ -228,7 +228,7 @@ def merge_packages(existing_package, new_package_data, replace=False): return updated_fields -def merge_or_create_package(scanned_package, visit_level, override=False): +def merge_or_create_package(scanned_package, visit_level, override=False, filename=None): """ Update Package from ``scanned_package`` instance if `visit_level` is greater than the mining level of the existing package. @@ -348,10 +348,13 @@ def merge_or_create_package(scanned_package, visit_level, override=False): existing_related_package = existing_related_packages.first() package_content = scanned_package.extra_data.get("package_content") + if not filename: + filename = fileutils.file_name(package_uri) + package_data = dict( # FIXME: we should get the file_name in the # PackageData object instead. - filename=fileutils.file_name(package_uri), + filename=filename, # TODO: update the PackageDB model release_date=scanned_package.release_date, mining_level=mining_level, diff --git a/minecode/tests/collectors/test_bitbucket.py b/minecode/tests/collectors/test_bitbucket.py new file mode 100644 index 00000000..304af4ee --- /dev/null +++ b/minecode/tests/collectors/test_bitbucket.py @@ -0,0 +1,41 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +from django.test import TestCase as DjangoTestCase + +from minecode.collectors import bitbucket +from minecode.utils_test import JsonBasedTesting + + +class BitbucketPriorityQueueTests(JsonBasedTesting, DjangoTestCase): + test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles") + + def test_bitbucket_get_all_package_version_author(self): + repo_path = "creachadair/stringset" + version_author_list = bitbucket.bitbucket_get_all_package_version_author(repo_path) + expected = [ + ("v0.0.1", "M. J. Fromberger"), + ("v0.0.10", "M. J. Fromberger"), + ("v0.0.11", "M. J. Fromberger"), + ("v0.0.12", "M. J. Fromberger"), + ("v0.0.13", "M. J. Fromberger"), + ("v0.0.14", "M. J. Fromberger"), + ("v0.0.2", "M. J. Fromberger"), + ("v0.0.3", "M. J. Fromberger"), + ("v0.0.4", "M. J. Fromberger"), + ("v0.0.5", "M. J. Fromberger"), + ("v0.0.6", "M. J. Fromberger"), + ("v0.0.7", "M. J. Fromberger"), + ("v0.0.8", "M. J. Fromberger"), + ("v0.0.9", "M. J. Fromberger"), + ] + for item in version_author_list: + self.assertIn(item, expected) diff --git a/minecode/tests/collectors/test_cargo.py b/minecode/tests/collectors/test_cargo.py index 63ccb3aa..ebaab2fb 100644 --- a/minecode/tests/collectors/test_cargo.py +++ b/minecode/tests/collectors/test_cargo.py @@ -41,7 +41,7 @@ def test_get_package_json(self): self.assertListEqual(list(keys), expected_list) self.assertEqual(json_contents["crate"]["id"], "sam") - def test_map_npm_package(self): + def test_map_cargo_package(self): package_count = packagedb.models.Package.objects.all().count() self.assertEqual(0, package_count) package_url = PackageURL.from_string("pkg:cargo/sam@0.3.1") diff --git a/minecode/tests/collectors/test_github.py b/minecode/tests/collectors/test_github.py new file mode 100644 index 00000000..dbb345fc --- /dev/null +++ b/minecode/tests/collectors/test_github.py @@ -0,0 +1,39 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +from django.test import TestCase as DjangoTestCase + +from minecode.collectors import github +from minecode.utils_test import JsonBasedTesting + + +class GithubPriorityQueueTests(JsonBasedTesting, DjangoTestCase): + test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles") + + def test_github_get_all_versions(self): + repo_path = "aboutcode-org/purldb" + versions = github.github_get_all_versions(repo_path) + expected = [ + "v7.0.0", + "v6.0.0", + "v5.0.1", + "v5.0.0", + "v3.0.0", + "v2.0.0", + "purldb-toolkit-v0.1.0", + "purl2vcs-v2.0.0", + "purl2vcs-v1.0.2", + "pre-scan-queue-update", + "matchcode-toolkit-v3.0.0", + "matchcode-toolkit-v1.1.1", + ] + for item in versions: + self.assertIn(item, expected) diff --git a/minecode/tests/collectors/test_gitlab.py b/minecode/tests/collectors/test_gitlab.py new file mode 100644 index 00000000..5bfedbd5 --- /dev/null +++ b/minecode/tests/collectors/test_gitlab.py @@ -0,0 +1,33 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +from django.test import TestCase as DjangoTestCase + +from minecode.collectors import gitlab +from minecode.utils_test import JsonBasedTesting + + +class GitlabPriorityQueueTests(JsonBasedTesting, DjangoTestCase): + test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles") + + def test_gitlab_get_all_package_version_author(self): + repo_path = "xx_network%2Fprimitives" + version_author_list = gitlab.gitlab_get_all_package_version_author(repo_path) + expected = [ + ("v0.0.5", "Richard T. Carback III", "rick.carback@gmail.com"), + ("v0.0.4", "Richard T. Carback III", "rick.carback@gmail.com"), + ("v0.0.3", "Benjamin Wenger", "ben@privategrity.com"), + ("v0.0.2", "Richard T. Carback III", "rick.carback@gmail.com"), + ("v0.0.1", "Jonathan Wenger", "jono@elixxir.io"), + ("v0.0.0", "Sydney Anne Erickson", "sydney@elixxir.io"), + ] + for item in version_author_list: + self.assertIn(item, expected) diff --git a/minecode/tests/collectors/test_golang.py b/minecode/tests/collectors/test_golang.py new file mode 100644 index 00000000..dea621a8 --- /dev/null +++ b/minecode/tests/collectors/test_golang.py @@ -0,0 +1,125 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import os + +from django.test import TestCase as DjangoTestCase + +from packageurl import PackageURL + +import packagedb +from minecode.collectors import golang +from minecode.utils_test import JsonBasedTesting + + +class GoLangPriorityQueueTests(JsonBasedTesting, DjangoTestCase): + test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles") + + def test_extract_golang_subset_purl(self): + test1 = "pkg:golang/github.com/rickar/cal/v2@2.1.23" + test2 = "pkg:golang/github.com/rickar/cal/v2" + expected_path1 = "rickar/cal" + expected_version1 = "2.1.23" + + result_path1, result_version1 = golang.extract_golang_subset_purl(test1) + result_path2, result_version2 = golang.extract_golang_subset_purl(test2) + + self.assertEqual(expected_path1, result_path1) + self.assertEqual(result_version1, expected_version1) + + self.assertEqual(expected_path1, result_path2) + self.assertEqual(result_version2, "") + + def test_gitlab_updated_purl(self): + test1 = "pkg:golang/gitlab.com/gitlab-org/api/client-go@0.127.0" + test2 = "pkg:golang/gitlab.com/gitlab-org/api/client-go" + expected_path1 = "gitlab-org%2Fapi%2Fclient-go" + expected_version1 = "0.127.0" + + result_path1, result_version1 = golang.gitlab_updated_purl(test1) + result_path2, result_version2 = golang.gitlab_updated_purl(test2) + + self.assertEqual(expected_path1, result_path1) + self.assertEqual(result_version1, expected_version1) + + self.assertEqual(expected_path1, result_path2) + self.assertEqual(result_version2, "") + + def test_get_package_json_gitlab(self): + json_contents = golang.get_package_json("xx_network%2Fprimitives", "gitlab") + expected_id = 20321795 + expected_name = "primitives" + + self.assertEqual(json_contents.get("id"), expected_id) + self.assertEqual(json_contents.get("name"), expected_name) + + def test_get_package_json_bitbucket(self): + json_contents = golang.get_package_json("lebronto_kerovol/gwerror", "bitbucket") + expected_full_name = "lebronto_kerovol/gwerror" + expected_name = "gwerror" + + self.assertEqual(json_contents.get("full_name"), expected_full_name) + self.assertEqual(json_contents.get("name"), expected_name) + + def test_map_go_package_gitlab(self): + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(0, package_count) + package_url = PackageURL.from_string( + "pkg:golang/gitlab.com/gitlab-org/api/client-go@0.127.0" + ) + + with open(self.get_test_loc("golang/client-go_0.127.0.json")) as file: + package_json = json.load(file) + golang.map_golang_package(package_url, package_json, ("test_pipeline")) + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(1, package_count) + package = packagedb.models.Package.objects.all().first() + expected_purl_str = "pkg:golang/gitlab.com/gitlab-org/api/client-go@0.127.0" + expected_download_url = "https://gitlab.com/api/v4/projects/gitlab-org%2Fapi%2Fclient-go/repository/archive.zip?sha=v0.127.0" + self.assertEqual(expected_purl_str, package.purl) + self.assertEqual(expected_download_url, package.download_url) + + def test_map_go_package_bitbucket(self): + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(0, package_count) + package_url = PackageURL.from_string("pkg:golang/bitbucket.org/digi-sense/gg-core@0.3.64") + + with open(self.get_test_loc("golang/gg-core_0.3.64.json")) as file: + package_json = json.load(file) + golang.map_golang_package(package_url, package_json, ("test_pipeline")) + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(1, package_count) + package = packagedb.models.Package.objects.all().first() + expected_purl_str = "pkg:golang/bitbucket.org/digi-sense/gg-core@0.3.64" + expected_download_url = "https://bitbucket.org/digi-sense/gg-core/get/v0.3.64.zip" + self.assertEqual(expected_purl_str, package.purl) + self.assertEqual(expected_download_url, package.download_url) + + def test_map_go_package_others(self): + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(0, package_count) + package_url = PackageURL.from_string("pkg:golang/golang.org/x/oauth2@0.29.0") + + with open(self.get_test_loc("golang/oauth2_0.29.0.json")) as file: + package_json = json.load(file) + golang.map_golang_package(package_url, package_json, ("test_pipeline")) + package_count = packagedb.models.Package.objects.all().count() + self.assertEqual(1, package_count) + package = packagedb.models.Package.objects.all().first() + expected_purl_str = "pkg:golang/golang.org/x/oauth2@0.29.0" + expected_download_url = "https://proxy.golang.org/golang.org/x/oauth2/@v/v0.29.0.zip" + self.assertEqual(expected_purl_str, package.purl) + self.assertEqual(expected_download_url, package.download_url) + + def test_process_download_metadata(self): + url = "https://bitbucket.org/digi-sense/gg-core/get/v0.3.64.zip" + _package_json, filename = golang.process_download_metadata(url, {}) + exprected_filename = "digi-sense-gg-core-9d3dfdc43161.zip" + self.assertEqual(exprected_filename, filename) diff --git a/minecode/tests/testfiles/golang/client-go_0.127.0.json b/minecode/tests/testfiles/golang/client-go_0.127.0.json new file mode 100644 index 00000000..bf4d2f99 --- /dev/null +++ b/minecode/tests/testfiles/golang/client-go_0.127.0.json @@ -0,0 +1,41 @@ +{ + "id": 65271576, + "description": "GitLab Go SDK\r\n\r\nThis project has been migrated from `github.com/xanzy/go-gitlab`.", + "name": "client-go", + "name_with_namespace": "GitLab.org / API / client-go", + "path": "client-go", + "path_with_namespace": "gitlab-org/api/client-go", + "created_at": "2024-12-10T14:14:37.357Z", + "default_branch": "main", + "tag_list": [ + "gitlab", + "go", + "sdk" + ], + "topics": [ + "gitlab", + "go", + "sdk" + ], + "ssh_url_to_repo": "git@gitlab.com:gitlab-org/api/client-go.git", + "http_url_to_repo": "https://gitlab.com/gitlab-org/api/client-go.git", + "web_url": "https://gitlab.com/gitlab-org/api/client-go", + "readme_url": "https://gitlab.com/gitlab-org/api/client-go/-/blob/main/README.md", + "forks_count": 36, + "avatar_url": null, + "star_count": 54, + "last_activity_at": "2025-04-15T04:41:12.062Z", + "namespace": { + "id": 98461187, + "name": "API", + "path": "api", + "kind": "group", + "full_path": "gitlab-org/api", + "parent_id": 9970, + "avatar_url": null, + "web_url": "https://gitlab.com/groups/gitlab-org/api" + }, + "download_url": "https://gitlab.com/api/v4/projects/gitlab-org%2Fapi%2Fclient-go/repository/archive.zip?sha=v0.127.0", + "author": "Patrick Rice", + "email": "patrick.rice@kingland.com" + } diff --git a/minecode/tests/testfiles/golang/gg-core_0.3.64.json b/minecode/tests/testfiles/golang/gg-core_0.3.64.json new file mode 100644 index 00000000..2e5c6fef --- /dev/null +++ b/minecode/tests/testfiles/golang/gg-core_0.3.64.json @@ -0,0 +1,130 @@ +{ + "type": "repository", + "full_name": "digi-sense/gg-core", + "links": { + "self": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core" + }, + "html": { + "href": "https://bitbucket.org/digi-sense/gg-core" + }, + "avatar": { + "href": "https://bytebucket.org/ravatar/%7B1bc29c37-3104-464d-ba27-bffa8847bcd6%7D?ts=3152224" + }, + "pullrequests": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/pullrequests" + }, + "commits": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/commits" + }, + "forks": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/forks" + }, + "watchers": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/watchers" + }, + "branches": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/refs/branches" + }, + "tags": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/refs/tags" + }, + "downloads": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/downloads" + }, + "source": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/src" + }, + "clone": [ + { + "name": "https", + "href": "https://bitbucket.org/digi-sense/gg-core.git" + }, + { + "name": "ssh", + "href": "git@bitbucket.org:digi-sense/gg-core.git" + } + ], + "hooks": { + "href": "https://api.bitbucket.org/2.0/repositories/digi-sense/gg-core/hooks" + } + }, + "name": "gg-core", + "slug": "gg-core", + "description": "", + "scm": "git", + "website": null, + "owner": { + "display_name": "Digi Sense", + "links": { + "self": { + "href": "https://api.bitbucket.org/2.0/workspaces/%7B2b6cccfa-727a-4d9a-82e0-bd8a163847a0%7D" + }, + "avatar": { + "href": "https://bitbucket.org/account/digi-sense/avatar/" + }, + "html": { + "href": "https://bitbucket.org/%7B2b6cccfa-727a-4d9a-82e0-bd8a163847a0%7D/" + } + }, + "type": "team", + "uuid": "{2b6cccfa-727a-4d9a-82e0-bd8a163847a0}", + "username": "digi-sense" + }, + "workspace": { + "type": "workspace", + "uuid": "{2b6cccfa-727a-4d9a-82e0-bd8a163847a0}", + "name": "Digi Sense", + "slug": "digi-sense", + "links": { + "avatar": { + "href": "https://bitbucket.org/workspaces/digi-sense/avatar/?ts=1732269844" + }, + "html": { + "href": "https://bitbucket.org/digi-sense/" + }, + "self": { + "href": "https://api.bitbucket.org/2.0/workspaces/digi-sense" + } + } + }, + "is_private": false, + "project": { + "type": "project", + "key": "BASE", + "uuid": "{32414877-738a-4002-96a4-893a906d88c8}", + "name": "base", + "links": { + "self": { + "href": "https://api.bitbucket.org/2.0/workspaces/digi-sense/projects/BASE" + }, + "html": { + "href": "https://bitbucket.org/digi-sense/workspace/projects/BASE" + }, + "avatar": { + "href": "https://bitbucket.org/digi-sense/workspace/projects/BASE/avatar/32?ts=1632311978" + } + } + }, + "fork_policy": "allow_forks", + "created_on": "2022-07-08T14:40:46.166913+00:00", + "updated_on": "2025-04-14T16:01:39.490611+00:00", + "size": 47007088, + "language": "go", + "uuid": "{1bc29c37-3104-464d-ba27-bffa8847bcd6}", + "mainbranch": { + "name": "master", + "type": "branch" + }, + "override_settings": { + "default_merge_strategy": true, + "branching_model": true + }, + "parent": null, + "enforced_signed_commits": null, + "has_issues": false, + "has_wiki": false, + "repo_workspace_name": "digi-sense/gg-core", + "download_url": "https://bitbucket.org/digi-sense/gg-core/get/v0.3.64.zip", + "author": "" + } diff --git a/minecode/tests/testfiles/golang/oauth2_0.29.0.json b/minecode/tests/testfiles/golang/oauth2_0.29.0.json new file mode 100644 index 00000000..d06c80a0 --- /dev/null +++ b/minecode/tests/testfiles/golang/oauth2_0.29.0.json @@ -0,0 +1,21 @@ +{ + "versionKey": { + "system": "GO", + "name": "golang.org/x/oauth2", + "version": "v0.29.0" + }, + "publishedAt": "2025-03-19T22:59:26Z", + "isDefault": "FALSE", + "licenses": ["BSD-3-Clause"], + "advisoryKeys": [], + "links": [ + { + "label": "SOURCE_REPO", + "url": "https://go.googlesource.com/oauth2" + } + ], + "slsaProvenances": [], + "attestations": [], + "registries": [], + "relatedProjects": [] +} diff --git a/packagedb/api.py b/packagedb/api.py index f168df31..73fc4d91 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -978,6 +978,7 @@ def _reindex_package(package, reindexed_packages, **kwargs): "pypi", "cargo", "gem", + "golang", "nuget", ] diff --git a/setup.cfg b/setup.cfg index f401ad33..f3fda32b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ install_requires = jawa >= 2.2.0 markdown >= 3.8.2 natsort >= 8.4.0 - packageurl-python >= 0.17.1 + packageurl-python >= 0.17.3 psycopg[binary] >= 3.2.9 PyGithub >= 2.6.1 reppy2 >= 0.3.6 @@ -61,8 +61,8 @@ install_requires = urlpy >= 0.5 matchcode-toolkit >= 7.2.2 purl2vcs >= 2.0.0 - univers >= 30.12.1 - scancodeio >= 35.0.0 + univers >= 31.0.0 + scancodeio >= 35.3.0 GitPython >= 3.1.44 samecode >= 0.5.1 # FederatedCode integration