From 6f6924fc7595f95d13134fd4b173011e2b083e4d Mon Sep 17 00:00:00 2001 From: Gang Li Date: Tue, 17 Dec 2024 09:11:05 +0800 Subject: [PATCH 01/64] chore: add --version flag to support version check Signed-off-by: Gang Li --- charon/cmd/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index 16a0129d..e2f54677 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from click import group +from click import group, version_option, pass_context from charon.cmd.cmd_upload import upload from charon.cmd.cmd_delete import delete from charon.cmd.cmd_index import index @@ -22,7 +22,9 @@ @group() -def cli(): +@version_option() +@pass_context +def cli(ctx): """Charon is a tool to synchronize several types of artifacts repository data to Red Hat Ronda service (maven.repository.redhat.com). From b7f3d5334beffabbe711ad381791fba592009a59 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 9 May 2025 16:31:30 +0800 Subject: [PATCH 02/64] Fix mmeng-4362: re-sort the indexing page items --- charon/pkgs/indexing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index 4d50e036..4710cdab 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -23,7 +23,7 @@ from jinja2 import Template import os import logging -from typing import List, Set, Dict +from typing import List, Dict from charon.utils.strings import remove_prefix @@ -48,7 +48,7 @@ def __get_index_template(package_type: str) -> str: class IndexedHTML(object): # object for holding index html file data - def __init__(self, title: str, header: str, items: Set[str]): + def __init__(self, title: str, header: str, items: List[str]): self.title = title self.header = header self.items = items @@ -174,8 +174,8 @@ def __to_html_content(package_type: str, contents: List[str], folder: str) -> st items = temp_items else: items.extend(contents) - items_set = set(__sort_index_items(items)) - index = IndexedHTML(title=folder, header=folder, items=items_set) + items_result = list(filter(lambda c: c.strip(), __sort_index_items(set(items)))) + index = IndexedHTML(title=folder, header=folder, items=items_result) return index.generate_index_file_content(package_type) @@ -303,8 +303,8 @@ def re_index( real_contents.append(c) else: real_contents = contents - logger.debug(real_contents) index_content = __to_html_content(package_type, real_contents, path) + logger.debug("The re-indexed page content: %s", index_content) if not dry_run: index_path = os.path.join(path, "index.html") if path == "/": From 6578fea3db15f1b7ea724149a8d2ef63cf9aa9a8 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 12 May 2025 13:57:22 +0800 Subject: [PATCH 03/64] Fix pip warning: add pyproject.toml --- pyproject.toml | 108 +++++++++++++++++++++++++++++++++++++++++ tests/requirements.txt | 1 - 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..43ab9cb4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,108 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "setuptools-scm"] + +[project] +name = "charon" +version = "1.3.3" +authors = [ + {name = "RedHat EXD SPMM"}, +] +readme = "README.md" +keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] +license-files = ["LICENSE"] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 1 - Planning", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Topic :: Software Development :: Build Tools", + "Topic :: Utilities", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "Jinja2>=3.1.3", + "boto3>=1.18.35", + "botocore>=1.21.35", + "click>=8.1.3", + "requests>=2.25.0", + "PyYAML>=5.4.1", + "defusedxml>=0.7.1", + "subresource-integrity>=0.2", + "jsonschema>=4.9.1", + "urllib3>=1.25.10", + "semantic-version>=2.10.0" +] + +[project.optional-dependencies] +dev = [ + "pylint", + "flake8", + "pep8", + "mypy", + "tox", +] +test = [ + "flexmock>=0.10.6", + "responses>=0.9.0", + "pytest<=7.1.3", + "pytest-cov", + "pytest-html", + "requests-mock", + "moto>=5.0.16,<6", + "python-gnupg>=0.5.0,<1" +] + +[project.scripts] +charon = "charon.cmd:cli" + +[tool.setuptools] +packages = ["charon"] + +[tool.setuptools_scm] +fallback_version = "1.3.4+dev.fallback" + +[tool.setuptools.package-data] +charon = ["schemas/*.json"] + +[tool.mypy] +python_version = "3.9" + +[tool.coverage.report] +skip_covered = true +show_missing = true +fail_under = 90 +exclude_lines = [ + "def __repr__", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "return NotImplemented", +] + +[tool.pytest.ini_options] +log_cli_level = "DEBUG" +log_format = "%(asctime)s %(levelname)s %(message)s" +log_date_format = "%Y-%m-%d %H:%M:%S" +testpaths = [ + "tests", +] + +[tool.flake8] +show_source = true +ignore = [ + "D100", # missing docstring in public module + "D104", # missing docstring in public package + "D105", # missing docstring in magic method + "W503", # line break before binary operator + "E203", # whitespace before ':' + "E501", # line too long + "E731", # do not assign a lambda expression +] +per-file-ignores = [ + "tests/*:D101,D102,D103", # missing docstring in public class, method, function +] \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 09f63266..408de626 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,7 +3,6 @@ responses>=0.9.0 pytest<=7.1.3 pytest-cov pytest-html -flake8 requests-mock moto>=5.0.16,<6 python-gnupg>=0.5.0,<1 From 54aeae8908845ad7ffce3808da460ff4eb4e247e Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 28 Apr 2025 21:47:11 +0800 Subject: [PATCH 04/64] RADAS: add radas configurations --- charon/config.py | 69 +++++++++++ charon/schemas/charon.json | 38 +++++++ tests/test_config_radas.py | 226 +++++++++++++++++++++++++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 tests/test_config_radas.py diff --git a/charon/config.py b/charon/config.py index 86f826ea..35efe6eb 100644 --- a/charon/config.py +++ b/charon/config.py @@ -24,6 +24,69 @@ logger = logging.getLogger(__name__) +class RadasConfig(object): + def __init__(self, data: Dict): + self.__umb_host: str = data.get("umb_host", None) + self.__umb_host_port: str = data.get("umb_host_port", "5671") + self.__result_queue: str = data.get("result_queue", None) + self.__request_queue: str = data.get("request_queue", None) + self.__client_ca: str = data.get("client_ca", None) + self.__client_key: str = data.get("client_key", None) + self.__client_key_pass_file: str = data.get("client_key_pass_file", None) + self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + + def validate(self) -> bool: + if not self.__umb_host: + logger.error("Missing host name setting for UMB!") + return False + if not self.__result_queue: + logger.error("Missing the queue setting to receive siging result in UMB!") + return False + if not self.__request_queue: + logger.error("Missing the queue setting to send signing request in UMB!") + return False + if self.__client_ca and not os.access(self.__client_ca, os.R_OK): + logger.error("The client CA file is not valid!") + return False + if self.__client_key and not os.access(self.__client_key, os.R_OK): + logger.error("The client key file is not valid!") + return False + if self.__client_key_pass_file and not os.access(self.__client_key_pass_file, os.R_OK): + logger.error("The client key password file is not valid!") + return False + if self.__root_ca and not os.access(self.__root_ca, os.R_OK): + logger.error("The root ca file is not valid!") + return False + return True + + def umb_target(self) -> str: + return f'amqps://{self.__umb_host}:{self.__umb_host_port}' + + def result_queue(self) -> str: + return self.__result_queue + + def request_queue(self) -> str: + return self.__request_queue + + def client_ca(self) -> str: + return self.__client_ca + + def client_key(self) -> str: + return self.__client_key + + def client_key_password(self) -> str: + pass_file = self.__client_key_pass_file + if os.access(pass_file, os.R_OK): + with open(pass_file, 'r') as f: + return f.read() + elif pass_file: + logger.warning("The key password file is not accessible. Will ignore the password.") + return "" + + def root_ca(self) -> str: + return self.__root_ca + + class CharonConfig(object): """CharonConfig is used to store all configurations for charon tools. @@ -39,6 +102,9 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + radas_config: Dict = data.get("radas", None) + if radas_config: + self.__radas_config__: RadasConfig = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -67,6 +133,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def get_radas_config(self) -> RadasConfig: + return self.__radas_config__ + def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath diff --git a/charon/schemas/charon.json b/charon/schemas/charon.json index f6a931d1..3ffde818 100644 --- a/charon/schemas/charon.json +++ b/charon/schemas/charon.json @@ -30,6 +30,44 @@ "type": "string", "description": "signature command to be used for signature" }, + "radas": { + "type": "object", + "descrition": "", + "properties": { + "umb_host": { + "type": "string", + "description": "The host of UMB" + }, + "umb_host_port": { + "type": "string", + "description": "The port of UMB host" + }, + "result_queue": { + "type": "string", + "description": "The queue in UMB to receive radas signing result" + }, + "request_queue": { + "type": "string", + "description": "The queue in UMB to send signing request to RADAS" + }, + "client_ca": { + "type": "string", + "description": "the client ca file path" + }, + "client_key": { + "type": "string", + "description": "the client key file path" + }, + "client_key_pass_file":{ + "type": "string", + "description": "the file contains password of the client key" + }, + "root_ca": { + "type": "string", + "description": "the root ca file path" + } + } + }, "targets": { "type": "object", "patternProperties": { diff --git a/tests/test_config_radas.py b/tests/test_config_radas.py new file mode 100644 index 00000000..152dc1c2 --- /dev/null +++ b/tests/test_config_radas.py @@ -0,0 +1,226 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import unittest +import os +import charon.config as config +import shutil +import tempfile +from tests.base import BaseTest +from charon.utils.files import overwrite_file + + +class RadasConfigTest(unittest.TestCase): + def setUp(self) -> None: + self.__base = BaseTest() + self.__prepare_ca() + + def tearDown(self) -> None: + self.__base.tearDown() + self.__clear_ca() + + def test_full_radas_config(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + root_ca: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, + self.__client_key_pass_file, self.__root_ca) + print(radas_settings) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertTrue(rconf.validate()) + + def test_missing_umb_host(self): + radas_settings = """ +radas: + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_missing_result_queue(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_missing_request_queue(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_ca(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_ca_path) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_key(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_key_path) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_password_file(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_root_ca(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + root_ca: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, + self.__client_key_pass_file, self.__root_ca) + os.remove(self.__root_ca) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def __change_config_content(self, content: str): + self.__base.change_home() + config_base = self.__base.get_config_base() + os.mkdir(config_base) + self.__base.prepare_config(config_base, content) + + def __prepare_ca(self): + self.__tempdir = tempfile.mkdtemp() + self.__client_ca_path = os.path.join(self.__tempdir, "client_ca.crt") + self.__client_key_path = os.path.join(self.__tempdir, "client_key.crt") + self.__client_key_pass_file = os.path.join(self.__tempdir, "client_key_password.txt") + self.__root_ca = os.path.join(self.__tempdir, "root_ca.crt") + overwrite_file(self.__client_ca_path, "client ca") + overwrite_file(self.__client_key_path, "client key") + overwrite_file(self.__client_key_pass_file, "it's password") + overwrite_file(self.__root_ca, "root ca") + + def __clear_ca(self): + shutil.rmtree(self.__tempdir) From e322b4eb6344c7fdc7c3f9a7561ca7e39546ca61 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 13 May 2025 11:48:00 +0800 Subject: [PATCH 05/64] Feat: Support to accept the response of signing result from RADAS --- charon/constants.py | 3 + charon/pkgs/maven.py | 35 ++++- charon/pkgs/oras_client.py | 64 ++++++++ charon/pkgs/radas_signature_handler.py | 210 +++++++++++++++++++++++++ requirements.txt | 2 + 5 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 charon/pkgs/oras_client.py create mode 100644 charon/pkgs/radas_signature_handler.py diff --git a/charon/constants.py b/charon/constants.py index 6751aecd..4c8320db 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,3 +175,6 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" +DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" +DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 +DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 \ No newline at end of file diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9f50f35b..900cc4cb 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,6 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature +import charon.pkgs.radas_signature_handler as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix @@ -408,11 +409,35 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if contain_signature is set to True - if gen_sign: - conf = get_config(config) - if not conf: - sys.exit(1) + # 10. Generate signature file if radas sign is enabled, or do detached sign if contain_signature is set to True + conf = get_config(config) + if not conf: + sys.exit(1) + + if conf.get_radas_sign_enabled(): + logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) + if not _generated_signs: + logger.error( + "No sign result files were downloaded, " + "please make sure the sign process is already done and without timeout") + return (tmp_root, False) + + failed_metas.extend(_failed_metas) + generated_signs.extend(_generated_signs) + logger.info("Singature generation against radas done.\n") + + logger.info("Start upload radas singature files to s3 bucket %s\n", bucket_name) + _failed_metas = s3_client.upload_signatures( + meta_file_paths=generated_signs, + target=(bucket_name, prefix), + product=None, + root=top_level + ) + failed_metas.extend(_failed_metas) + logger.info("Signature files uploading against radas done.\n") + + elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) command = conf.get_detach_signature_command() artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))] diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py new file mode 100644 index 00000000..fca62006 --- /dev/null +++ b/charon/pkgs/oras_client.py @@ -0,0 +1,64 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oras.client +import logging +from charon.config import get_config +from typing import List + +logger = logging.getLogger(__name__) + +class OrasClient: + """ + Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. + """ + + def __init__(self): + self.conf = get_config() + self.client = oras.client.OrasClient() + + def login_if_needed(self) -> None: + """ + If quay_radas_auth_enabled is true, call login to authenticate. + """ + + if self.conf and self.conf.is_quay_radas_auth_enabled(): + logger.info("Logging in to registry.") + res = self.client.login( + hostname=self.conf.get_quay_radas_registry(), + username=self.conf.get_quay_radas_username(), + password=self.conf.get_quay_radas_password(), + ) + logger.info(res) + else: + logger.info("Registry auth not enabled, skip login.") + + def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: + """ + Call oras‑py’s pull method to pull the remote file to local. + Args: + result_reference_url (str): Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): Local save path (e.g. “/tmp/sign”). + """ + files = [] + try: + self.login_if_needed() + # the filename should be possibly named by the digest hash value based on the oras source code + files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) + logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) + except Exception as e: + logger.error("Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e) + finally: + return files \ No newline at end of file diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py new file mode 100644 index 00000000..5bacdab7 --- /dev/null +++ b/charon/pkgs/radas_signature_handler.py @@ -0,0 +1,210 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import proton +import proton.handlers +import threading +import logging +import json +import os +import asyncio +from typing import List, Any, Tuple, Callable, Dict +from charon.config import get_config +from charon.constants import DEFAULT_SIGN_RESULT_LOC +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_COUNT +from charon.constants import DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC +from charon.pkgs.oras_client import OrasClient + +logger = logging.getLogger(__name__) + +class SignHandler: + """ + Handle the sign result status management + """ + _is_processing: bool = True + _downloaded_files: List[str] = [] + + @classmethod + def is_processing(cls) -> bool: + return cls._is_processing + + @classmethod + def get_downloaded_files(cls) -> List[str]: + return cls._downloaded_files.copy() + + @classmethod + def set_processing(cls, value: bool) -> None: + cls._is_processing = value + + @classmethod + def set_downloaded_files(cls, files: List[str]) -> None: + cls._downloaded_files = files + +class UmbListener(proton.handlers.MessagingHandler): + """ + UmbListener class (AMQP version), register this when setup UmbClient + """ + + def __init__(self) -> None: + super().__init__() + + def on_start(self, event: proton.Event) -> None: + """ + On start callback + """ + conf = get_config() + if not conf: + sys.exit(1) + event.container.create_receiver(conf.get_amqp_queue()) + + def on_message(self, event: proton.Event) -> None: + """ + On message callback + """ + # handle response from radas in a thread + thread = threading.Thread( + target=self._process_message, + args=[event.message.body] + ) + thread.start() + + def on_error(self, event: proton.Event) -> None: + """ + On error callback + """ + logger.error("Received an error event:\n%s", event) + + def on_disconnected(self, event: proton.Event) -> None: + """ + On disconnected callback + """ + logger.error("Disconnected from AMQP broker.") + + def _process_message(msg: Any) -> None: + """ + Process a message received from UMB + Args: + msg: The message body received + """ + try: + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") + + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return + + conf = get_config() + if not conf: + sign_result_loc = DEFAULT_SIGN_RESULT_LOC + sign_result_loc = os.getenv("SIGN_RESULT_LOC") or conf.get_sign_result_loc() + logger.info("Using SIGN RESULT LOC: %s", sign_result_loc) + + sign_result_parent_dir = os.path.dirname(sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) + + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, + sign_result_loc=sign_result_loc + ) + SignHandler.set_downloaded_files(files) + finally: + SignHandler.set_processing(False) + +def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: + """ + Generate .asc files based on RADAS sign result json file + """ + conf = get_config() + timeout_count = conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + wait_interval_sec = conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + wait_count = 0 + while SignHandler.is_processing(): + wait_count += 1 + if wait_count > timeout_count: + logger.warning("Timeout when waiting for sign response.") + break + time.sleep(wait_interval_sec) + + files = SignHandler.get_downloaded_files() + if not files: + return [], [] + + # should only have the single sign result json file from the radas registry + json_file_path = files[0] + try: + with open(json_file_path, 'r') as f: + data = json.load(f) + except Exception as e: + logger.error(f"Failed to read or parse the JSON file: {e}") + raise + + async def generate_single_sign_file( + file_path: str, signature: str, failed_paths: List[str], generated_signs: List[str], + sem: asyncio.BoundedSemaphore + ): + async with sem: + if not file_path or not signature: + logger.error(f"Invalid JSON entry") + return + # remove the root path maven-repository + filename = file_path.split("/", 1)[1] + signature = item.get("signature") + + artifact_path = os.path.join(top_level, filename) + asc_filename = f"{filename}.asc" + signature_path = os.path.join(top_level, asc_filename) + + if not os.path.isfile(artifact_path): + logger.warning("Artifact missing, skip signature file generation") + return + + try: + with open(signature_path, 'w') as asc_file: + asc_file.write(signature) + generated_signs.append(signature_path) + logger.info(f"Generated .asc file: {signature_path}") + except Exception as e: + failed_paths.append(signature_path) + logger.error(f"Failed to write .asc file for {artifact_path}: {e}") + + result = data.get("result", []) + return __do_path_cut_and( + path_handler=generate_single_sign_file, + data=result + ) + +def __do_path_cut_and( + path_handler: Callable, + data: List[Dict[str, str]] +) -> Tuple[List[str], List[str]]: + + failed_paths: List[str] = [] + generated_signs: List[str] = [] + tasks = [] + sem = asyncio.BoundedSemaphore(10) + for item in data: + file_path = item.get("file") + signature = item.get("signature") + tasks.append( + asyncio.ensure_future( + path_handler(file_path, signature, failed_paths, generated_signs, sem) + ) + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete(asyncio.gather(*tasks)) + return (failed_paths, generated_signs) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7919fc27..75bb4b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 +oras>=0.2.31 +python-qpid-proton>=0.39.0 \ No newline at end of file From 07e43f37ed067d2853865bb0c7eb491b895f0a70 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 13 May 2025 13:31:44 +0800 Subject: [PATCH 06/64] Format Code and fix typo --- charon/config.py | 6 +-- charon/constants.py | 2 +- charon/pkgs/maven.py | 5 ++- charon/pkgs/oras_client.py | 16 ++++--- charon/pkgs/radas_signature_handler.py | 58 ++++++++++++++------------ 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/charon/config.py b/charon/config.py index 35efe6eb..c9cbdc59 100644 --- a/charon/config.py +++ b/charon/config.py @@ -141,14 +141,12 @@ def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath if not config_file_path or not os.path.isfile(config_file_path): config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE) - data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json') + data = read_yaml_from_file_path(config_file_path, "schemas/charon.json") return CharonConfig(data) def get_template(template_file: str) -> str: - template = os.path.join( - os.getenv("HOME", ''), ".charon/template", template_file - ) + template = os.path.join(os.getenv("HOME", ""), ".charon/template", template_file) if os.path.isfile(template): with open(template, encoding="utf-8") as file_: return file_.read() diff --git a/charon/constants.py b/charon/constants.py index 4c8320db..c8b8d125 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -177,4 +177,4 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 -DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 \ No newline at end of file +DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 900cc4cb..8c14d822 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -409,12 +409,13 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if radas sign is enabled, or do detached sign if contain_signature is set to True + # 10. Generate signature file if radas sign is enabled, + # or do detached sign if contain_signature is set to True conf = get_config(config) if not conf: sys.exit(1) - if conf.get_radas_sign_enabled(): + if conf.is_radas_sign_enabled(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) if not _generated_signs: diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py index fca62006..95310215 100644 --- a/charon/pkgs/oras_client.py +++ b/charon/pkgs/oras_client.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import oras.client import logging from charon.config import get_config @@ -20,6 +21,7 @@ logger = logging.getLogger(__name__) + class OrasClient: """ Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. @@ -49,16 +51,18 @@ def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ Call oras‑py’s pull method to pull the remote file to local. Args: - result_reference_url (str): Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). - sign_result_loc (str): Local save path (e.g. “/tmp/sign”). + result_reference_url (str): + Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”). """ files = [] try: self.login_if_needed() - # the filename should be possibly named by the digest hash value based on the oras source code files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) except Exception as e: - logger.error("Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e) - finally: - return files \ No newline at end of file + logger.error( + "Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e + ) + return files diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 5bacdab7..9120bc9b 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import proton import proton.handlers import threading @@ -20,6 +21,8 @@ import json import os import asyncio +import sys +import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config from charon.constants import DEFAULT_SIGN_RESULT_LOC @@ -29,10 +32,12 @@ logger = logging.getLogger(__name__) + class SignHandler: """ Handle the sign result status management """ + _is_processing: bool = True _downloaded_files: List[str] = [] @@ -52,6 +57,7 @@ def set_processing(cls, value: bool) -> None: def set_downloaded_files(cls, files: List[str]) -> None: cls._downloaded_files = files + class UmbListener(proton.handlers.MessagingHandler): """ UmbListener class (AMQP version), register this when setup UmbClient @@ -74,10 +80,7 @@ def on_message(self, event: proton.Event) -> None: On message callback """ # handle response from radas in a thread - thread = threading.Thread( - target=self._process_message, - args=[event.message.body] - ) + thread = threading.Thread(target=self._process_message, args=[event.message.body]) thread.start() def on_error(self, event: proton.Event) -> None: @@ -103,8 +106,8 @@ def _process_message(msg: Any) -> None: result_reference_url = msg_dict.get("result_reference") if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return + logger.warning("Not found result_reference in message，ignore.") + return conf = get_config() if not conf: @@ -117,20 +120,24 @@ def _process_message(msg: Any) -> None: oras_client = OrasClient() files = oras_client.pull( - result_reference_url=result_reference_url, - sign_result_loc=sign_result_loc + result_reference_url=result_reference_url, sign_result_loc=sign_result_loc ) SignHandler.set_downloaded_files(files) finally: SignHandler.set_processing(False) + def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ conf = get_config() - timeout_count = conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT - wait_interval_sec = conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + timeout_count = ( + conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + ) + wait_interval_sec = ( + conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + ) wait_count = 0 while SignHandler.is_processing(): wait_count += 1 @@ -146,23 +153,25 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: # should only have the single sign result json file from the radas registry json_file_path = files[0] try: - with open(json_file_path, 'r') as f: + with open(json_file_path, "r") as f: data = json.load(f) except Exception as e: - logger.error(f"Failed to read or parse the JSON file: {e}") + logger.error("Failed to read or parse the JSON file: %s", e) raise async def generate_single_sign_file( - file_path: str, signature: str, failed_paths: List[str], generated_signs: List[str], - sem: asyncio.BoundedSemaphore + file_path: str, + signature: str, + failed_paths: List[str], + generated_signs: List[str], + sem: asyncio.BoundedSemaphore, ): async with sem: if not file_path or not signature: - logger.error(f"Invalid JSON entry") + logger.error("Invalid JSON entry") return # remove the root path maven-repository filename = file_path.split("/", 1)[1] - signature = item.get("signature") artifact_path = os.path.join(top_level, filename) asc_filename = f"{filename}.asc" @@ -173,23 +182,20 @@ async def generate_single_sign_file( return try: - with open(signature_path, 'w') as asc_file: + with open(signature_path, "w") as asc_file: asc_file.write(signature) generated_signs.append(signature_path) - logger.info(f"Generated .asc file: {signature_path}") + logger.info("Generated .asc file: %s", signature_path) except Exception as e: failed_paths.append(signature_path) - logger.error(f"Failed to write .asc file for {artifact_path}: {e}") + logger.error("Failed to write .asc file for %s: %s", artifact_path, e) result = data.get("result", []) - return __do_path_cut_and( - path_handler=generate_single_sign_file, - data=result - ) + return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) + def __do_path_cut_and( - path_handler: Callable, - data: List[Dict[str, str]] + path_handler: Callable, data: List[Dict[str, str]] ) -> Tuple[List[str], List[str]]: failed_paths: List[str] = [] @@ -207,4 +213,4 @@ def __do_path_cut_and( loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) - return (failed_paths, generated_signs) \ No newline at end of file + return (failed_paths, generated_signs) From 42dbb01f8711362807985c0b9495bb7126ea0dc9 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 19 May 2025 15:12:26 +0800 Subject: [PATCH 07/64] Rename radas_sign_timeout_retry_count and radas_sign_timeout_retry_interval --- charon/config.py | 15 +++++++++++++-- charon/constants.py | 4 ++-- charon/pkgs/radas_signature_handler.py | 19 +++++++++++-------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/charon/config.py b/charon/config.py index c9cbdc59..b6eb07cd 100644 --- a/charon/config.py +++ b/charon/config.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging import os from typing import Dict, List, Optional @@ -34,6 +35,10 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) + self.__radas_sign_timeout_retry_interval: int = data.get( + "radas_sign_timeout_retry_interval", 60 + ) def validate(self) -> bool: if not self.__umb_host: @@ -60,7 +65,7 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f'amqps://{self.__umb_host}:{self.__umb_host_port}' + return f"amqps://{self.__umb_host}:{self.__umb_host_port}" def result_queue(self) -> str: return self.__result_queue @@ -77,7 +82,7 @@ def client_key(self) -> str: def client_key_password(self) -> str: pass_file = self.__client_key_pass_file if os.access(pass_file, os.R_OK): - with open(pass_file, 'r') as f: + with open(pass_file, "r") as f: return f.read() elif pass_file: logger.warning("The key password file is not accessible. Will ignore the password.") @@ -86,6 +91,12 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def radas_sign_timeout_retry_count(self) -> int: + return self.__radas_sign_timeout_retry_count + + def radas_sign_timeout_retry_interval(self) -> int: + return self.__radas_sign_timeout_retry_interval + class CharonConfig(object): """CharonConfig is used to store all configurations for charon diff --git a/charon/constants.py b/charon/constants.py index c8b8d125..25970c36 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -176,5 +176,5 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" -DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 -DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 9120bc9b..eb755086 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -26,8 +26,8 @@ from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config from charon.constants import DEFAULT_SIGN_RESULT_LOC -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_COUNT -from charon.constants import DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient logger = logging.getLogger(__name__) @@ -132,19 +132,22 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: Generate .asc files based on RADAS sign result json file """ conf = get_config() - timeout_count = ( - conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + rconf = conf.get_radas_config() if conf else None + timeout_retry_count = ( + rconf.radas_sign_timeout_retry_count() if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT ) - wait_interval_sec = ( - conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + timeout_retry_interval = ( + rconf.radas_sign_timeout_retry_interval() + if conf + else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL ) wait_count = 0 while SignHandler.is_processing(): wait_count += 1 - if wait_count > timeout_count: + if wait_count > timeout_retry_count: logger.warning("Timeout when waiting for sign response.") break - time.sleep(wait_interval_sec) + time.sleep(timeout_retry_interval) files = SignHandler.get_downloaded_files() if not files: From 89be82f359f724903f5d759530bf5399f20e8882 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 19 May 2025 21:00:36 +0800 Subject: [PATCH 08/64] Use oras registry_config and registry url parse to finalize login --- charon/config.py | 11 +++++++++++ charon/pkgs/oras_client.py | 22 +++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/charon/config.py b/charon/config.py index b6eb07cd..fe3c1174 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,6 +35,9 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__quay_radas_registry_config: str = data.get( + "quay_radas_registry_config", os.path.join(os.getenv("HOME", ""), ".oras/config.json") + ) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 @@ -62,6 +65,11 @@ def validate(self) -> bool: if self.__root_ca and not os.access(self.__root_ca, os.R_OK): logger.error("The root ca file is not valid!") return False + if self.__quay_radas_registry_config and not os.access( + self.__quay_radas_registry_config, os.R_OK + ): + logger.error("The quay registry config for oras is not valid!") + return False return True def umb_target(self) -> str: @@ -91,6 +99,9 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def quay_radas_registry_config(self) -> str: + return self.__quay_radas_registry_config + def radas_sign_timeout_retry_count(self) -> int: return self.__radas_sign_timeout_retry_count diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py index 95310215..b5446def 100644 --- a/charon/pkgs/oras_client.py +++ b/charon/pkgs/oras_client.py @@ -18,6 +18,7 @@ import logging from charon.config import get_config from typing import List +from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -31,21 +32,24 @@ def __init__(self): self.conf = get_config() self.client = oras.client.OrasClient() - def login_if_needed(self) -> None: + def login_if_needed(self, registry: str) -> None: """ - If quay_radas_auth_enabled is true, call login to authenticate. + If quay_radas_registry_config is provided, call login to authenticate. """ + if not registry.startswith("http://") and not registry.startswith("https://"): + registry = "https://" + registry + registry = urlparse(registry).netloc - if self.conf and self.conf.is_quay_radas_auth_enabled(): - logger.info("Logging in to registry.") + rconf = self.conf.get_radas_config() if self.conf else None + if rconf and rconf.quay_radas_registry_config(): + logger.info("Logging in to registry: %s", registry) res = self.client.login( - hostname=self.conf.get_quay_radas_registry(), - username=self.conf.get_quay_radas_username(), - password=self.conf.get_quay_radas_password(), + hostname=registry, + config_path=rconf.quay_radas_registry_config(), ) logger.info(res) else: - logger.info("Registry auth not enabled, skip login.") + logger.info("Registry config is not provided, skip login.") def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ @@ -58,7 +62,7 @@ def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ files = [] try: - self.login_if_needed() + self.login_if_needed(registry=result_reference_url) files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) except Exception as e: From d4eee680354d62fab7caca8f283426cbf345bd5e Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 09:32:24 +0800 Subject: [PATCH 09/64] Change sign_result_loc to cmd flag both for sign request and maven upload --- charon/cmd/cmd_upload.py | 16 +++- charon/config.py | 7 +- charon/constants.py | 1 - charon/pkgs/maven.py | 15 ++-- charon/pkgs/radas_signature_handler.py | 112 +++++++++++-------------- 5 files changed, 78 insertions(+), 73 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a867df01..d56a644d 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -136,6 +136,16 @@ default=False ) @option("--dryrun", "-n", is_flag=True, default=False) +@option( + "--sign_result_loc", + "-l", + default="/tmp/sign", + help=""" + The local save path for oras to pull the radas signature result. + Sign request will use this path to download the signature result, + Upload will use the file on this path to generate the corresponding .asc files + """, +) @command() def upload( repo: str, @@ -150,7 +160,8 @@ def upload( sign_key: str = "redhatdevel", debug=False, quiet=False, - dryrun=False + dryrun=False, + sign_result_loc="/tmp/sign" ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -221,7 +232,8 @@ def upload( key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, - config=config + config=config, + sign_result_loc=sign_result_loc ) if not succeeded: sys.exit(1) diff --git a/charon/config.py b/charon/config.py index fe3c1174..ce418b33 100644 --- a/charon/config.py +++ b/charon/config.py @@ -48,7 +48,7 @@ def validate(self) -> bool: logger.error("Missing host name setting for UMB!") return False if not self.__result_queue: - logger.error("Missing the queue setting to receive siging result in UMB!") + logger.error("Missing the queue setting to receive signing result in UMB!") return False if not self.__request_queue: logger.error("Missing the queue setting to send signing request in UMB!") @@ -124,8 +124,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + self.__radas_config_enable: bool = data.get("radas_config_enable", False) radas_config: Dict = data.get("radas", None) if radas_config: + self.__radas_config_enable = True self.__radas_config__: RadasConfig = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: @@ -155,6 +157,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_config_enable(self) -> bool: + return self.__radas_config_enable + def get_radas_config(self) -> RadasConfig: return self.__radas_config__ diff --git a/charon/constants.py b/charon/constants.py index 25970c36..35ea560a 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,6 +175,5 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" -DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 8c14d822..dde1a4cb 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -275,7 +275,8 @@ def handle_maven_uploading( key=None, dry_run=False, manifest_bucket_name=None, - config=None + config=None, + sign_result_loc="/tmp/sign" ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -415,9 +416,11 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_sign_enabled(): + if conf.is_radas_config_enable(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) - (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( + top_level=top_level, sign_result_loc=sign_result_loc + ) if not _generated_signs: logger.error( "No sign result files were downloaded, " @@ -426,9 +429,9 @@ def handle_maven_uploading( failed_metas.extend(_failed_metas) generated_signs.extend(_generated_signs) - logger.info("Singature generation against radas done.\n") + logger.info("Radas signature files generation done.\n") - logger.info("Start upload radas singature files to s3 bucket %s\n", bucket_name) + logger.info("Start upload radas signature files to s3 bucket %s\n", bucket_name) _failed_metas = s3_client.upload_signatures( meta_file_paths=generated_signs, target=(bucket_name, prefix), @@ -436,7 +439,7 @@ def handle_maven_uploading( root=top_level ) failed_metas.extend(_failed_metas) - logger.info("Signature files uploading against radas done.\n") + logger.info("Radas signature files uploading done.\n") elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index eb755086..9ef00d8f 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -14,8 +14,6 @@ limitations under the License. """ -import proton -import proton.handlers import threading import logging import json @@ -25,57 +23,40 @@ import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config -from charon.constants import DEFAULT_SIGN_RESULT_LOC from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient +from proton import Event +from proton.handlers import MessagingHandler logger = logging.getLogger(__name__) -class SignHandler: - """ - Handle the sign result status management - """ - - _is_processing: bool = True - _downloaded_files: List[str] = [] - - @classmethod - def is_processing(cls) -> bool: - return cls._is_processing - - @classmethod - def get_downloaded_files(cls) -> List[str]: - return cls._downloaded_files.copy() - - @classmethod - def set_processing(cls, value: bool) -> None: - cls._is_processing = value - - @classmethod - def set_downloaded_files(cls, files: List[str]) -> None: - cls._downloaded_files = files - - -class UmbListener(proton.handlers.MessagingHandler): +class UmbListener(MessagingHandler): """ UmbListener class (AMQP version), register this when setup UmbClient + Attributes: + sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, + this value transfers from the cmd flag, should register UmbListener when the client starts """ - def __init__(self) -> None: + def __init__(self, sign_result_loc: str) -> None: super().__init__() + self.sign_result_loc = sign_result_loc - def on_start(self, event: proton.Event) -> None: + def on_start(self, event: Event) -> None: """ On start callback """ conf = get_config() - if not conf: + rconf = conf.get_radas_config() if conf else None + if not rconf: sys.exit(1) - event.container.create_receiver(conf.get_amqp_queue()) + conn = event.container.connect(rconf.umb_target()) + event.container.create_receiver(conn, rconf.result_queue()) + logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) - def on_message(self, event: proton.Event) -> None: + def on_message(self, event: Event) -> None: """ On message callback """ @@ -83,51 +64,43 @@ def on_message(self, event: proton.Event) -> None: thread = threading.Thread(target=self._process_message, args=[event.message.body]) thread.start() - def on_error(self, event: proton.Event) -> None: + def on_connection_error(self, event: Event) -> None: """ - On error callback + On connection error callback """ logger.error("Received an error event:\n%s", event) - def on_disconnected(self, event: proton.Event) -> None: + def on_disconnected(self, event: Event) -> None: """ On disconnected callback """ logger.error("Disconnected from AMQP broker.") - def _process_message(msg: Any) -> None: + def _process_message(self, msg: Any) -> None: """ Process a message received from UMB Args: msg: The message body received """ - try: - msg_dict = json.loads(msg) - result_reference_url = msg_dict.get("result_reference") + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") - if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return - - conf = get_config() - if not conf: - sign_result_loc = DEFAULT_SIGN_RESULT_LOC - sign_result_loc = os.getenv("SIGN_RESULT_LOC") or conf.get_sign_result_loc() - logger.info("Using SIGN RESULT LOC: %s", sign_result_loc) + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return - sign_result_parent_dir = os.path.dirname(sign_result_loc) - os.makedirs(sign_result_parent_dir, exist_ok=True) + logger.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) - oras_client = OrasClient() - files = oras_client.pull( - result_reference_url=result_reference_url, sign_result_loc=sign_result_loc - ) - SignHandler.set_downloaded_files(files) - finally: - SignHandler.set_processing(False) + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) -def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: +def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ @@ -138,21 +111,34 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: ) timeout_retry_interval = ( rconf.radas_sign_timeout_retry_interval() - if conf + if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL ) wait_count = 0 - while SignHandler.is_processing(): + + # Wait until files appear in the sign_result_loc directory + while True: + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] + if files: # If files exist, break the loop + break + wait_count += 1 if wait_count > timeout_retry_count: logger.warning("Timeout when waiting for sign response.") break time.sleep(timeout_retry_interval) - files = SignHandler.get_downloaded_files() if not files: return [], [] + if len(files) > 1: + logger.error("Multiple files found in %s. Expected only one file.", sign_result_loc) + return [], [] + # should only have the single sign result json file from the radas registry json_file_path = files[0] try: From d2e86c969fa5bd23e244e9d7e8999258e6d152a7 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 10:22:18 +0800 Subject: [PATCH 10/64] Update quay_radas_registry_config to default None since it's not necessary for public --- charon/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charon/config.py b/charon/config.py index ce418b33..ca238e63 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,9 +35,7 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") - self.__quay_radas_registry_config: str = data.get( - "quay_radas_registry_config", os.path.join(os.getenv("HOME", ""), ".oras/config.json") - ) + self.__quay_radas_registry_config: str = data.get("quay_radas_registry_config", None) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 From 75740e5a952156109734fe75a6f15477f71fa174 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 11:12:06 +0800 Subject: [PATCH 11/64] Use radas_config validate instead of is_radas_config_enable option --- charon/config.py | 10 +++------- charon/pkgs/maven.py | 3 ++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/charon/config.py b/charon/config.py index ca238e63..a53bdaa6 100644 --- a/charon/config.py +++ b/charon/config.py @@ -122,11 +122,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) - self.__radas_config_enable: bool = data.get("radas_config_enable", False) + self.__radas_config__: Optional[RadasConfig] = None radas_config: Dict = data.get("radas", None) if radas_config: - self.__radas_config_enable = True - self.__radas_config__: RadasConfig = RadasConfig(radas_config) + self.__radas_config__ = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -155,10 +154,7 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable - def is_radas_config_enable(self) -> bool: - return self.__radas_config_enable - - def get_radas_config(self) -> RadasConfig: + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index dde1a4cb..3971fd06 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -416,7 +416,8 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_config_enable(): + rconf = conf.get_radas_config() + if rconf and rconf.validate(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( top_level=top_level, sign_result_loc=sign_result_loc From 947f23ce5434e0131c32e3825e8f566eca2f9753 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 14:12:41 +0800 Subject: [PATCH 12/64] Ignore the registry config if the provided config path is not valid to read --- charon/config.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/charon/config.py b/charon/config.py index a53bdaa6..5e72f270 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,7 +35,9 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") - self.__quay_radas_registry_config: str = data.get("quay_radas_registry_config", None) + self.__quay_radas_registry_config: Optional[str] = data.get( + "quay_radas_registry_config", None + ) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 @@ -66,8 +68,10 @@ def validate(self) -> bool: if self.__quay_radas_registry_config and not os.access( self.__quay_radas_registry_config, os.R_OK ): - logger.error("The quay registry config for oras is not valid!") - return False + self.__quay_radas_registry_config = None + logger.warning( + "The quay registry config for oras is not valid, will ignore the registry config!" + ) return True def umb_target(self) -> str: @@ -97,7 +101,7 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca - def quay_radas_registry_config(self) -> str: + def quay_radas_registry_config(self) -> Optional[str]: return self.__quay_radas_registry_config def radas_sign_timeout_retry_count(self) -> int: From da803ddddc2d62b6d443fb2f7effbb9f19165c5e Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 15:43:39 +0800 Subject: [PATCH 13/64] Add is_radas_enabled unified method to check radas enablement --- charon/config.py | 3 +++ charon/pkgs/maven.py | 3 +-- charon/pkgs/radas_signature_handler.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charon/config.py b/charon/config.py index 5e72f270..2995ffdf 100644 --- a/charon/config.py +++ b/charon/config.py @@ -158,6 +158,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_enabled(self) -> bool: + return bool(self.__radas_config__ and self.__radas_config__.validate()) + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 3971fd06..4ca1be0d 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -416,8 +416,7 @@ def handle_maven_uploading( if not conf: sys.exit(1) - rconf = conf.get_radas_config() - if rconf and rconf.validate(): + if conf.is_radas_enabled(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( top_level=top_level, sign_result_loc=sign_result_loc diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 9ef00d8f..27a14548 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -49,8 +49,12 @@ def on_start(self, event: Event) -> None: On start callback """ conf = get_config() - rconf = conf.get_radas_config() if conf else None - if not rconf: + if not (conf and conf.is_radas_enabled()): + sys.exit(1) + + rconf = conf.get_radas_config() + # explicit check to pass the type checker + if rconf is None: sys.exit(1) conn = event.container.connect(rconf.umb_target()) event.container.create_receiver(conn, rconf.result_queue()) From f784b1627df28806ba9762fd91d6bc9bd5718cea Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 15:20:27 +0800 Subject: [PATCH 14/64] Change on_message process method without using threads --- charon/pkgs/radas_signature_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 27a14548..a62a0d96 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -14,7 +14,6 @@ limitations under the License. """ -import threading import logging import json import os @@ -64,9 +63,7 @@ def on_message(self, event: Event) -> None: """ On message callback """ - # handle response from radas in a thread - thread = threading.Thread(target=self._process_message, args=[event.message.body]) - thread.start() + self._process_message(event.message.body) def on_connection_error(self, event: Event) -> None: """ From 57ae2224791b32a2789031a175c0138a221afd4a Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 28 Apr 2025 21:47:11 +0800 Subject: [PATCH 15/64] RADAS: Add sign command skeleton --- charon/cmd/__init__.py | 4 ++ charon/cmd/cmd_sign.py | 148 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 charon/cmd/cmd_sign.py diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index e2f54677..985d7f79 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -19,6 +19,7 @@ from charon.cmd.cmd_index import index from charon.cmd.cmd_checksum import init_checksum, checksum from charon.cmd.cmd_cache import init_cf, cf +from charon.cmd.cmd_sign import sign @group() @@ -43,3 +44,6 @@ def cli(ctx): # init checksum command init_checksum() cli.add_command(checksum) + +# radas sign cmd +cli.add_command(sign) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py new file mode 100644 index 00000000..9578183e --- /dev/null +++ b/charon/cmd/cmd_sign.py @@ -0,0 +1,148 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List + +from charon.config import get_config, RadasConfig +from charon.cmd.internal import ( + _decide_mode, _safe_delete +) +from click import command, option, argument + +import traceback +import logging +import sys +import datetime + +logger = logging.getLogger(__name__) + + +@argument( + "repo_url", + type=str +) +@option( + "--requester", + "-r", + help=""" + The requester who sends the signing request. + """ +) +@option( + "--result_path", + "-p", + help=""" + The path which will save the sign result file. + """ +) +@option( + "--ignore_patterns", + "-i", + multiple=True, + help=""" + The regex patterns list to filter out the files which should + not be allowed to upload to S3. Can accept more than one pattern. + """ +) +@option( + "--work_dir", + "-w", + help=""" + The temporary working directory into which archives should + be extracted, when needed. + """ +) +@option( + "--config", + "-c", + help=""" + The charon configuration yaml file path. Default is + $HOME/.charon/charon.yaml + """ +) +@option( + "--sign_key", + "-k", + help=""" + rpm-sign key to be used, will replace {{ key }} in default configuration for signature. + Does noting if detach_signature_command does not contain {{ key }} field. + """ +) +@option( + "--debug", + "-D", + help="Debug mode, will print all debug logs for problem tracking.", + is_flag=True, + default=False +) +@option( + "--quiet", + "-q", + help="Quiet mode, will shrink most of the logs except warning and errors.", + is_flag=True, + default=False +) +@command() +def sign( + repo_url: str, + requester: str, + result_path: str, + ignore_patterns: List[str] = None, + work_dir: str = None, + config: str = None, + sign_key: str = "redhatdevel", + debug=False, + quiet=False, + dryrun=False +): + """Do signing against files in the repo zip in repo_url through + radas service. The repo_url points to the maven zip repository + in quay.io, which will be sent as the source of the signing. + """ + tmp_dir = work_dir + logger.debug("%s", ignore_patterns) + try: + current = datetime.datetime.now().strftime("%Y%m%d%I%M") + _decide_mode("radas_sign", current, is_quiet=quiet, is_debug=debug) + if dryrun: + logger.info("Running in dry-run mode, no files will signed.") + conf = get_config(config) + if not conf: + logger.error("The charon configuration is not valid!") + sys.exit(1) + radas_conf = conf.get_radas_config() + if not radas_conf or not radas_conf.validate(): + logger.error("The configuration for radas is not valid!") + sys.exit(1) + sign_in_radas(repo_url, requester, sign_key, result_path, radas_conf) + except Exception: + print(traceback.format_exc()) + sys.exit(2) # distinguish between exception and bad config or bad state + finally: + if not debug and tmp_dir: + _safe_delete(tmp_dir) + + +def sign_in_radas(repo_url: str, + requester: str, + sign_key: str, + result_path: str, + radas_config: RadasConfig): + '''This function will be responsible to do the overall controlling of the whole process, + like trigger the send and register the receiver, and control the wait and timeout there. + ''' + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," + "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) + logger.info("Not implemented yet!") From 83a92f99ea1b529df329c835e738c47dd960afda Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 17:29:44 +0800 Subject: [PATCH 16/64] Remove timeout retry handling for sign result fetch from maven upload --- charon/pkgs/radas_signature_handler.py | 35 ++++---------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index a62a0d96..c04f0bbf 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,11 +19,8 @@ import os import asyncio import sys -import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient from proton import Event from proton.handlers import MessagingHandler @@ -105,33 +102,11 @@ def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str] """ Generate .asc files based on RADAS sign result json file """ - conf = get_config() - rconf = conf.get_radas_config() if conf else None - timeout_retry_count = ( - rconf.radas_sign_timeout_retry_count() if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT - ) - timeout_retry_interval = ( - rconf.radas_sign_timeout_retry_interval() - if rconf - else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL - ) - wait_count = 0 - - # Wait until files appear in the sign_result_loc directory - while True: - files = [ - os.path.join(sign_result_loc, f) - for f in os.listdir(sign_result_loc) - if os.path.isfile(os.path.join(sign_result_loc, f)) - ] - if files: # If files exist, break the loop - break - - wait_count += 1 - if wait_count > timeout_retry_count: - logger.warning("Timeout when waiting for sign response.") - break - time.sleep(timeout_retry_interval) + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] if not files: return [], [] From f90557c521ee19b68db0b3afa26896454dfaaa12 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 17:29:44 +0800 Subject: [PATCH 17/64] Some changes on sign cmd and config --- charon/cmd/cmd_sign.py | 32 +++++++++++++------------- charon/config.py | 12 ++++++---- charon/pkgs/radas_signature_handler.py | 16 ++++++++++--- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index 9578183e..629281cb 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -15,7 +15,8 @@ """ from typing import List -from charon.config import get_config, RadasConfig +from charon.config import get_config +from charon.pkgs.radas_signature_handler import sign_in_radas from charon.cmd.internal import ( _decide_mode, _safe_delete ) @@ -126,23 +127,22 @@ def sign( if not radas_conf or not radas_conf.validate(): logger.error("The configuration for radas is not valid!") sys.exit(1) - sign_in_radas(repo_url, requester, sign_key, result_path, radas_conf) + # All ignore files in global config should also be ignored in signing. + ig_patterns = conf.get_ignore_patterns() + if ignore_patterns: + ig_patterns.extend(ignore_patterns) + args = { + "repo_url": repo_url, + "requester": requester, + "sign_key": sign_key, + "result_path": result_path, + "ignore_patterns": ig_patterns, + "radas_config": radas_conf + } + sign_in_radas(**args) # type: ignore except Exception: print(traceback.format_exc()) - sys.exit(2) # distinguish between exception and bad config or bad state + sys.exit(2) finally: if not debug and tmp_dir: _safe_delete(tmp_dir) - - -def sign_in_radas(repo_url: str, - requester: str, - sign_key: str, - result_path: str, - radas_config: RadasConfig): - '''This function will be responsible to do the overall controlling of the whole process, - like trigger the send and register the receiver, and control the wait and timeout there. - ''' - logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," - "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) - logger.info("Not implemented yet!") diff --git a/charon/config.py b/charon/config.py index 2995ffdf..44bc9c77 100644 --- a/charon/config.py +++ b/charon/config.py @@ -126,10 +126,12 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) - self.__radas_config__: Optional[RadasConfig] = None radas_config: Dict = data.get("radas", None) if radas_config: - self.__radas_config__ = RadasConfig(radas_config) + self.__radas_config = RadasConfig(radas_config) + self.__radas_enabled = bool(self.__radas_config and self.__radas_config.validate()) + else: + self.__radas_enabled = False def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -159,10 +161,10 @@ def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable def is_radas_enabled(self) -> bool: - return bool(self.__radas_config__ and self.__radas_config__.validate()) + return self.__radas_enabled - def get_radas_config(self) -> Optional[RadasConfig]: - return self.__radas_config__ + def get_radas_config(self) -> RadasConfig: + return self.__radas_config def get_config(cfgPath=None) -> CharonConfig: diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index c04f0bbf..46794f49 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -20,7 +20,7 @@ import asyncio import sys from typing import List, Any, Tuple, Callable, Dict -from charon.config import get_config +from charon.config import get_config, RadasConfig from charon.pkgs.oras_client import OrasClient from proton import Event from proton.handlers import MessagingHandler @@ -28,9 +28,10 @@ logger = logging.getLogger(__name__) -class UmbListener(MessagingHandler): +class RadasReceiver(MessagingHandler): """ - UmbListener class (AMQP version), register this when setup UmbClient + This receiver will listen to UMB message queue to receive signing message for + signing result. Attributes: sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, this value transfers from the cmd flag, should register UmbListener when the client starts @@ -179,3 +180,12 @@ def __do_path_cut_and( loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) return (failed_paths, generated_signs) + + +def sign_in_radas(repo_url: str, + requester: str, + sign_key: str, + result_path: str, + ignore_patterns: List[str], + radas_config: RadasConfig): + logger.info("Start signing for %s", repo_url) From 9f0073fd2edf4d7b27044c29ac41fe4e665a550f Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 26 May 2025 20:35:28 +0800 Subject: [PATCH 18/64] Fix: add back the radas_config type for mypy check --- charon/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charon/config.py b/charon/config.py index 44bc9c77..5e7734d0 100644 --- a/charon/config.py +++ b/charon/config.py @@ -127,6 +127,7 @@ def __init__(self, data: Dict): self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) radas_config: Dict = data.get("radas", None) + self.__radas_config: Optional[RadasConfig] = None if radas_config: self.__radas_config = RadasConfig(radas_config) self.__radas_enabled = bool(self.__radas_config and self.__radas_config.validate()) @@ -163,7 +164,7 @@ def is_aws_cf_enable(self) -> bool: def is_radas_enabled(self) -> bool: return self.__radas_enabled - def get_radas_config(self) -> RadasConfig: + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config From 730ae43370290817158db5fe675a94b91e419296 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 27 May 2025 10:46:43 +0800 Subject: [PATCH 19/64] Add Unit tests for RADAS signing results parse and generation --- charon/pkgs/maven.py | 2 +- charon/pkgs/radas_signature_handler.py | 4 + tests/test_radas_sign_handler.py | 162 +++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 tests/test_radas_sign_handler.py diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 4ca1be0d..9d895365 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -423,7 +423,7 @@ def handle_maven_uploading( ) if not _generated_signs: logger.error( - "No sign result files were downloaded, " + "No sign result files were generated, " "please make sure the sign process is already done and without timeout") return (tmp_root, False) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 46794f49..5d25212d 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -103,6 +103,10 @@ def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str] """ Generate .asc files based on RADAS sign result json file """ + if not os.path.isdir(sign_result_loc): + logger.error("Sign result loc dir does not exist: %s", sign_result_loc) + return [], [] + files = [ os.path.join(sign_result_loc, f) for f in os.listdir(sign_result_loc) diff --git a/tests/test_radas_sign_handler.py b/tests/test_radas_sign_handler.py new file mode 100644 index 00000000..f0b90351 --- /dev/null +++ b/tests/test_radas_sign_handler.py @@ -0,0 +1,162 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import unittest +import tempfile +import os +import json +import shutil +import builtins +from unittest import mock +from charon.utils.files import overwrite_file +from charon.pkgs.radas_signature_handler import generate_radas_sign + +logger = logging.getLogger(__name__) + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.__prepare_sign_result_file() + + def tearDown(self) -> None: + super().tearDown() + self.__clear_sign_result_file() + + def test_multi_sign_files_generation(self): + self.__prepare_artifacts() + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(len(generated), 2) + self.assertIn(expected_asc1, generated) + self.assertIn(expected_asc2, generated) + + with open(expected_asc1) as f: + content1 = f.read() + with open(expected_asc2) as f: + content2 = f.read() + self.assertIn("signature1@hash", content1) + self.assertIn("signature2@hash", content2) + + def test_sign_files_generation_with_missing_artifacts(self): + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def test_sign_files_generation_with_failure(self): + self.__prepare_artifacts() + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + + # simulate expected_asc1 can not open to write properly + real_open = builtins.open + with mock.patch("builtins.open") as mock_open: + def side_effect(path, *args, **kwargs): + # this is for pylint check + mode = "r" + if len(args) > 0: + mode = args[0] + elif "mode" in kwargs: + mode = kwargs["mode"] + if path == expected_asc1 and "w" in mode: + raise IOError("mock write error") + return real_open(path, *args, **kwargs) + mock_open.side_effect = side_effect + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + + self.assertEqual(len(failed), 1) + self.assertNotIn(expected_asc1, generated) + self.assertIn(expected_asc2, generated) + + def test_sign_files_generation_with_missing_result(self): + self.__prepare_artifacts() + # simulate missing pull result by removing the sign result file loc + shutil.rmtree(self.__sign_result_loc) + + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def test_sign_files_generation_with_not_single_results(self): + self.__prepare_artifacts() + another_result_file = os.path.join(self.__sign_result_loc, "result2.json") + overwrite_file(another_result_file, "test_json") + + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def __prepare_sign_result_file(self): + self.__sign_result_loc = tempfile.mkdtemp() + self.__sign_result_file = os.path.join(self.__sign_result_loc, "result.json") + self.__repo_dir = os.path.join(tempfile.mkdtemp(), "maven-repository") + data = { + "request-id": "request-id", + "file-reference": "quay.io/org/maven-zip@hash", + "result": [ + { + "file": "maven-repository/foo/bar/1.0/foo-bar-1.0.jar", + "signature": ( + "-----BEGIN PGP SIGNATURE-----" + "signature1@hash" + "-----END PGP SIGNATURE-----" + ), + "checksum": "sha256:sha256-content", + }, + { + "file": "maven-repository/foo/bar/2.0/foo-bar-2.0.jar", + "signature": ( + "-----BEGIN PGP SIGNATURE-----" + "signature2@hash" + "-----END PGP SIGNATURE-----" + ), + "checksum": "sha256:sha256-content", + }, + ], + } + json_str = json.dumps(data, indent=2) + overwrite_file(self.__sign_result_file, json_str) + + def __prepare_artifacts(self): + os.makedirs(os.path.join(self.__repo_dir, "foo/bar/1.0"), exist_ok=True) + os.makedirs(os.path.join(self.__repo_dir, "foo/bar/2.0"), exist_ok=True) + artifact1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar") + artifact2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar") + with open(artifact1, "w") as f: + f.write("dummy1") + with open(artifact2, "w") as f: + f.write("dummy2") + + def __clear_sign_result_file(self): + if os.path.exists(self.__sign_result_loc): + shutil.rmtree(self.__sign_result_loc) + if os.path.exists(self.__repo_dir): + shutil.rmtree(self.__repo_dir) From 110279872edc07c92ce2af3ed87fcf2517dcff32 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 27 May 2025 11:08:33 +0800 Subject: [PATCH 20/64] Change 'result' to 'results' ref signing/radas-nonprod test samples --- charon/pkgs/radas_signature_handler.py | 2 +- tests/test_radas_sign_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 5d25212d..e670619c 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -160,7 +160,7 @@ async def generate_single_sign_file( failed_paths.append(signature_path) logger.error("Failed to write .asc file for %s: %s", artifact_path, e) - result = data.get("result", []) + result = data.get("results", []) return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) diff --git a/tests/test_radas_sign_handler.py b/tests/test_radas_sign_handler.py index f0b90351..b667eaea 100644 --- a/tests/test_radas_sign_handler.py +++ b/tests/test_radas_sign_handler.py @@ -121,7 +121,7 @@ def __prepare_sign_result_file(self): data = { "request-id": "request-id", "file-reference": "quay.io/org/maven-zip@hash", - "result": [ + "results": [ { "file": "maven-repository/foo/bar/1.0/foo-bar-1.0.jar", "signature": ( From 229f4487c3a1fd3ab6be405e789d5f2370c0dbb0 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 28 May 2025 09:36:32 +0800 Subject: [PATCH 21/64] Add request_id match logic for radas message receiver --- charon/pkgs/radas_signature_handler.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 46794f49..e89699d2 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -37,9 +37,10 @@ class RadasReceiver(MessagingHandler): this value transfers from the cmd flag, should register UmbListener when the client starts """ - def __init__(self, sign_result_loc: str) -> None: + def __init__(self, sign_result_loc: str, request_id: str) -> None: super().__init__() self.sign_result_loc = sign_result_loc + self.request_id = request_id def on_start(self, event: Event) -> None: """ @@ -82,8 +83,19 @@ def _process_message(self, msg: Any) -> None: msg: The message body received """ msg_dict = json.loads(msg) - result_reference_url = msg_dict.get("result_reference") + msg_request_id = msg_dict.get("request_id") + if msg_request_id != self.request_id: + logger.info( + "Message request_id %s does not match the request_id %s from sender, ignoring", + msg_request_id, + self.request_id, + ) + return + logger.info( + "Start to process the sign event message, request_id %s is matched", msg_request_id + ) + result_reference_url = msg_dict.get("result_reference") if not result_reference_url: logger.warning("Not found result_reference in message，ignore.") return From 2672bb3171a17c49f5949ff36ec71780edf4333d Mon Sep 17 00:00:00 2001 From: yma Date: Thu, 29 May 2025 11:48:45 +0800 Subject: [PATCH 22/64] Add sign response status and errors for receiver then use to control the main process --- charon/pkgs/radas_signature_handler.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 24543739..a21d75ec 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,7 +19,7 @@ import os import asyncio import sys -from typing import List, Any, Tuple, Callable, Dict +from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import get_config, RadasConfig from charon.pkgs.oras_client import OrasClient from proton import Event @@ -33,14 +33,23 @@ class RadasReceiver(MessagingHandler): This receiver will listen to UMB message queue to receive signing message for signing result. Attributes: - sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, - this value transfers from the cmd flag, should register UmbListener when the client starts + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”) for oras pull result, this value transfers + from the cmd flag,should register UmbListener when the client starts + request_id (str): + Identifier of the request for the signing result + sign_result_status (str): + Result of the signing(success/failed) + sign_result_errors (list): + Any errors encountered if signing fails, this will be empty list if successful """ def __init__(self, sign_result_loc: str, request_id: str) -> None: super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id + self.sign_result_status: Optional[str] = None + self.sign_result_errors: List[str] = [] def on_start(self, event: Event) -> None: """ @@ -95,6 +104,8 @@ def _process_message(self, msg: Any) -> None: logger.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) + self.sign_result_status = msg_dict.get("signing_status") + self.sign_result_errors = msg_dict.get("errors", []) result_reference_url = msg_dict.get("result_reference") if not result_reference_url: logger.warning("Not found result_reference in message，ignore.") From c7ebf3151481741019cce4a8e5ec77492596acf4 Mon Sep 17 00:00:00 2001 From: liyu Date: Thu, 29 May 2025 17:53:06 +0800 Subject: [PATCH 23/64] Feature of send radas sign request and unit test for it --- charon/pkgs/radas_signature_handler.py | 109 ++++++++++++++++++++++++- tests/test_radas_send_handler.py | 98 ++++++++++++++++++++++ 2 files changed, 203 insertions(+), 4 deletions(-) create mode 100644 tests/test_radas_send_handler.py diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index a21d75ec..6f862d07 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,11 +19,13 @@ import os import asyncio import sys +import uuid from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import get_config, RadasConfig from charon.pkgs.oras_client import OrasClient -from proton import Event +from proton import SSLDomain, Message, Event from proton.handlers import MessagingHandler +from proton.reactor import Container logger = logging.getLogger(__name__) @@ -48,6 +50,7 @@ def __init__(self, sign_result_loc: str, request_id: str) -> None: super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id + self.conn = None self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] @@ -63,8 +66,23 @@ def on_start(self, event: Event) -> None: # explicit check to pass the type checker if rconf is None: sys.exit(1) - conn = event.container.connect(rconf.umb_target()) - event.container.create_receiver(conn, rconf.result_queue()) + + ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) + ssl_domain.set_credentials( + rconf.client_ca(), + rconf.client_key(), + rconf.client_key_password() + ) + ssl_domain.set_trusted_ca_db(rconf.root_ca()) + ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + + self.conn = event.container.connect( + url=rconf.umb_target(), + ssl_domain=ssl_domain + ) + event.container.create_receiver( + self.conn, rconf.result_queue(), dynamic=True + ) logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) def on_message(self, event: Event) -> None: @@ -122,6 +140,60 @@ def _process_message(self, msg: Any) -> None: logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) +class RadasSender(MessagingHandler): + """ + This simple sender will send given string massage to UMB message queue to request signing. + Attributes: + payload (str): payload json string for radas to read, + this value construct from the cmd flag + """ + def __init__(self, payload: str): + super().__init__() + self.payload = payload + self.container = None + self.conn = None + self.sender = None + + def on_start(self, event): + """ + On start callback + """ + conf = get_config() + if not (conf and conf.is_radas_enabled()): + sys.exit(1) + + rconf = conf.get_radas_config() + if rconf is None: + sys.exit(1) + + ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) + ssl_domain.set_credentials( + rconf.client_ca(), + rconf.client_key(), + rconf.client_key_password() + ) + ssl_domain.set_trusted_ca_db(rconf.root_ca()) + ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + + self.container = event.container + self.conn = event.container.connect( + url=rconf.umb_target(), + ssl_domain=ssl_domain + ) + self.sender = event.container.create_sender(self.conn, rconf.request_queue()) + + def on_sendable(self): + """ + On message able to send callback + """ + request = self.payload + msg = Message(body=request) + if self.sender: + self.sender.send(msg) + if self.container: + self.container.stop() + + def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file @@ -215,4 +287,33 @@ def sign_in_radas(repo_url: str, result_path: str, ignore_patterns: List[str], radas_config: RadasConfig): - logger.info("Start signing for %s", repo_url) + """ + This function will be responsible to do the overall controlling of the whole process, + like trigger the send and register the receiver, and control the wait and timeout there. + """ + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," + "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) + request_id = str(uuid.uuid4()) + exclude = ignore_patterns if ignore_patterns else [] + + payload = { + "request_id": request_id, + "requested_by": requester, + "type": "mrrc", + "file_reference": repo_url, + "sig_keyname": sign_key, + "exclude": exclude + } + + listener = RadasReceiver(result_path, request_id) + sender = RadasSender(json.dumps(payload)) + + try: + Container(sender).run() + logger.info("Successfully sent signing request ID: %s", request_id) + Container(listener).run() + finally: + if listener.conn is not None: + listener.conn.close() + if sender.conn is not None: + sender.conn.close() diff --git a/tests/test_radas_send_handler.py b/tests/test_radas_send_handler.py new file mode 100644 index 00000000..b37fa05a --- /dev/null +++ b/tests/test_radas_send_handler.py @@ -0,0 +1,98 @@ +import tempfile +import os +from unittest import mock +import unittest +from charon.pkgs.radas_signature_handler import sign_in_radas + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_sign_in_radas_normal_flow(self): + with tempfile.TemporaryDirectory() as tmpdir: + # Mock configuration + mock_config = mock.MagicMock() + mock_config.is_radas_enabled.return_value = True + mock_radas_config = mock.MagicMock() + mock_config.get_radas_config.return_value = mock_radas_config + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_signature_handler.Container") as mock_container, \ + mock.patch( + "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ + mock.patch( + "charon.pkgs.radas_signature_handler.uuid.uuid4", return_value="mocked-uuid"): + + test_result_path = os.path.join(tmpdir, "results") + os.makedirs(test_result_path) + + sign_in_radas( + repo_url="quay.io/test/repo", + requester="test-user", + sign_key="test-key", + result_path=test_result_path, + ignore_patterns=[], + radas_config=mock_radas_config + ) + + # Verify Container.run() was called twice (sender and receiver) + self.assertEqual(mock_container.call_count, 2) + + # Verify request ID propagation + receiver_call = mock_container.call_args_list[1] + self.assertEqual(receiver_call.args[0].request_id, "mocked-uuid") + + def test_sign_in_radas_with_disabled_config(self): + mock_config = mock.MagicMock() + mock_config.is_radas_enabled.return_value = False + + with mock.patch( + "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ + self.assertRaises(SystemExit): + + sign_in_radas( + repo_url="quay.io/test/repo", + requester="test-user", + sign_key="test-key", + result_path="/tmp/results", + ignore_patterns=[], + radas_config=mock.MagicMock() + ) + + def test_sign_in_radas_connection_cleanup(self): + mock_config = mock.MagicMock() + mock_config.is_radas_enabled.return_value = True + mock_radas_config = mock.MagicMock() + + with mock.patch("charon.pkgs.radas_signature_handler.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_signature_handler.get_config", return_value=mock_config): + + mock_sender_conn = mock.MagicMock() + mock_listener_conn = mock.MagicMock() + + def container_side_effect(*args, **kwargs): + if args[0].__class__.__name__ == "RadasReceiver": + args[0].conn = mock_listener_conn + elif args[0].__class__.__name__ == "RadasSender": + args[0].conn = mock_sender_conn + return mock.MagicMock() + + mock_container.side_effect = container_side_effect + + sign_in_radas( + repo_url="quay.io/test/repo", + requester="test-user", + sign_key="test-key", + result_path="/tmp/results", + ignore_patterns=[], + radas_config=mock_radas_config + ) + + # Verify connections are closed + mock_sender_conn.close.assert_called_once() + mock_listener_conn.close.assert_called_once() From 678606bf5ed39a88984fe6e324c83b19eca633e1 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 30 May 2025 15:38:30 +0800 Subject: [PATCH 24/64] Refactor: refactor the RadasSender * rename radas_signature_handler.py to radas_sign.py * change RadasSender logic: * add retry logic * add error handling * avoid duplicate message sending * use a global RadasConfig instead created one in each Radas class --- charon/cmd/cmd_sign.py | 32 +-- charon/config.py | 18 +- charon/pkgs/maven.py | 2 +- ...das_signature_handler.py => radas_sign.py} | 209 ++++++++++-------- pyproject.toml | 3 +- requirements-dev.txt | 1 + setup.py | 3 +- tests/test_radas_send_handler.py | 98 -------- ...ndler.py => test_radas_sign_generation.py} | 2 +- tests/test_radas_sign_sender.py | 86 +++++++ 10 files changed, 236 insertions(+), 218 deletions(-) rename charon/pkgs/{radas_signature_handler.py => radas_sign.py} (61%) delete mode 100644 tests/test_radas_send_handler.py rename tests/{test_radas_sign_handler.py => test_radas_sign_generation.py} (99%) create mode 100644 tests/test_radas_sign_sender.py diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index 629281cb..a83a3e3f 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -16,10 +16,9 @@ from typing import List from charon.config import get_config -from charon.pkgs.radas_signature_handler import sign_in_radas -from charon.cmd.internal import ( - _decide_mode, _safe_delete -) +from charon.pkgs.radas_sign import sign_in_radas +from charon.cmd.internal import _decide_mode + from click import command, option, argument import traceback @@ -39,14 +38,16 @@ "-r", help=""" The requester who sends the signing request. - """ + """, + required=True ) @option( "--result_path", "-p", help=""" The path which will save the sign result file. - """ + """, + required=True ) @option( "--ignore_patterns", @@ -57,14 +58,6 @@ not be allowed to upload to S3. Can accept more than one pattern. """ ) -@option( - "--work_dir", - "-w", - help=""" - The temporary working directory into which archives should - be extracted, when needed. - """ -) @option( "--config", "-c", @@ -79,7 +72,8 @@ help=""" rpm-sign key to be used, will replace {{ key }} in default configuration for signature. Does noting if detach_signature_command does not contain {{ key }} field. - """ + """, + required=True ) @option( "--debug", @@ -100,10 +94,9 @@ def sign( repo_url: str, requester: str, result_path: str, + sign_key: str, ignore_patterns: List[str] = None, - work_dir: str = None, config: str = None, - sign_key: str = "redhatdevel", debug=False, quiet=False, dryrun=False @@ -112,7 +105,6 @@ def sign( radas service. The repo_url points to the maven zip repository in quay.io, which will be sent as the source of the signing. """ - tmp_dir = work_dir logger.debug("%s", ignore_patterns) try: current = datetime.datetime.now().strftime("%Y%m%d%I%M") @@ -139,10 +131,8 @@ def sign( "ignore_patterns": ig_patterns, "radas_config": radas_conf } + logger.debug("params: %s", args) sign_in_radas(**args) # type: ignore except Exception: print(traceback.format_exc()) sys.exit(2) - finally: - if not debug and tmp_dir: - _safe_delete(tmp_dir) diff --git a/charon/config.py b/charon/config.py index 5e7734d0..396e4be3 100644 --- a/charon/config.py +++ b/charon/config.py @@ -75,34 +75,36 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f"amqps://{self.__umb_host}:{self.__umb_host_port}" + return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" def result_queue(self) -> str: - return self.__result_queue + return self.__result_queue.strip() def request_queue(self) -> str: - return self.__request_queue + return self.__request_queue.strip() def client_ca(self) -> str: - return self.__client_ca + return self.__client_ca.strip() def client_key(self) -> str: - return self.__client_key + return self.__client_key.strip() def client_key_password(self) -> str: pass_file = self.__client_key_pass_file if os.access(pass_file, os.R_OK): with open(pass_file, "r") as f: - return f.read() + return f.read().strip() elif pass_file: logger.warning("The key password file is not accessible. Will ignore the password.") return "" def root_ca(self) -> str: - return self.__root_ca + return self.__root_ca.strip() def quay_radas_registry_config(self) -> Optional[str]: - return self.__quay_radas_registry_config + if self.__quay_radas_registry_config: + return self.__quay_radas_registry_config.strip() + return None def radas_sign_timeout_retry_count(self) -> int: return self.__radas_sign_timeout_retry_count diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9d895365..2f525ce3 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,7 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature -import charon.pkgs.radas_signature_handler as radas_signature +import charon.pkgs.radas_sign as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_sign.py similarity index 61% rename from charon/pkgs/radas_signature_handler.py rename to charon/pkgs/radas_sign.py index 6f862d07..355daa77 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_sign.py @@ -21,9 +21,9 @@ import sys import uuid from typing import List, Any, Tuple, Callable, Dict, Optional -from charon.config import get_config, RadasConfig +from charon.config import RadasConfig from charon.pkgs.oras_client import OrasClient -from proton import SSLDomain, Message, Event +from proton import SSLDomain, Message, Event, Sender from proton.handlers import MessagingHandler from proton.reactor import Container @@ -46,61 +46,42 @@ class RadasReceiver(MessagingHandler): Any errors encountered if signing fails, this will be empty list if successful """ - def __init__(self, sign_result_loc: str, request_id: str) -> None: + def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> None: super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id self.conn = None self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] - - def on_start(self, event: Event) -> None: - """ - On start callback - """ - conf = get_config() - if not (conf and conf.is_radas_enabled()): - sys.exit(1) - - rconf = conf.get_radas_config() - # explicit check to pass the type checker - if rconf is None: - sys.exit(1) - - ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) - ssl_domain.set_credentials( - rconf.client_ca(), - rconf.client_key(), - rconf.client_key_password() + self.rconf = rconf + self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self.ssl.set_trusted_ca_db(self.rconf.root_ca()) + self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self.ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() ) - ssl_domain.set_trusted_ca_db(rconf.root_ca()) - ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + def on_start(self, event: Event) -> None: self.conn = event.container.connect( - url=rconf.umb_target(), - ssl_domain=ssl_domain + url=self.rconf.umb_target(), + ssl_domain=self.ssl ) event.container.create_receiver( - self.conn, rconf.result_queue(), dynamic=True + self.conn, self.rconf.result_queue(), dynamic=True ) - logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) + logger.info("Listening on %s, queue: %s", + self.rconf.umb_target(), + self.rconf.result_queue()) def on_message(self, event: Event) -> None: - """ - On message callback - """ self._process_message(event.message.body) def on_connection_error(self, event: Event) -> None: - """ - On connection error callback - """ logger.error("Received an error event:\n%s", event) def on_disconnected(self, event: Event) -> None: - """ - On disconnected callback - """ logger.error("Disconnected from AMQP broker.") def _process_message(self, msg: Any) -> None: @@ -146,53 +127,106 @@ class RadasSender(MessagingHandler): Attributes: payload (str): payload json string for radas to read, this value construct from the cmd flag + rconf (RadasConfig): the configurations for the radas messaging + system. """ - def __init__(self, payload: str): - super().__init__() + def __init__(self, payload: Any, rconf: RadasConfig): + super(RadasSender, self).__init__() self.payload = payload - self.container = None - self.conn = None - self.sender = None - - def on_start(self, event): - """ - On start callback - """ - conf = get_config() - if not (conf and conf.is_radas_enabled()): - sys.exit(1) - - rconf = conf.get_radas_config() - if rconf is None: - sys.exit(1) - - ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) - ssl_domain.set_credentials( - rconf.client_ca(), - rconf.client_key(), - rconf.client_key_password() + self.rconf = rconf + self.message_sent = False # Flag to track if message was sent + self.status: Optional[str] = None + self.retried = 0 + self.pending: Optional[Message] = None + self.message: Optional[Message] = None + self.container: Optional[Container] = None + self.sender: Optional[Sender] = None + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") + self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self.ssl.set_trusted_ca_db(self.rconf.root_ca()) + self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self.ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() ) - ssl_domain.set_trusted_ca_db(rconf.root_ca()) - ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + def on_start(self, event): self.container = event.container - self.conn = event.container.connect( - url=rconf.umb_target(), - ssl_domain=ssl_domain + conn = self.container.connect( + url=self.rconf.umb_target(), + ssl_domain=self.ssl ) - self.sender = event.container.create_sender(self.conn, rconf.request_queue()) - - def on_sendable(self): - """ - On message able to send callback - """ - request = self.payload - msg = Message(body=request) + if conn: + self.sender = self.container.create_sender(conn, self.rconf.request_queue()) + + def on_sendable(self, event): + if not self.message_sent: + msg = Message(body=self.payload, durable=True) + self.log.debug("Sending message: %s to %s", msg.id, event.sender.target.address) + self._send_msg(msg) + self.message = msg + self.message_sent = True + + def on_error(self, event): + self.log.error("Error happened during message sending, reason %s", + event.description) + self.status = "failed" + + def on_rejected(self, event): + self.pending = self.message + self._handle_failed_delivery("Rejected") + + def on_released(self, event): + self.pending = self.message + self._handle_failed_delivery("Released") + + def on_accepted(self, event): + self.log.info("Message accepted by receiver: %s", event.delivery) + self.status = "success" + self.close() # Close connection after confirmation + + def on_timer_task(self, event): + message_to_retry = self.message + self._send_msg(message_to_retry) + self.pending = None + + def close(self): + self.log.info("Message has been sent successfully, close connection") if self.sender: - self.sender.send(msg) + self.sender.close() if self.container: self.container.stop() + def _send_msg(self, msg: Message): + if self.sender and self.sender.credit > 0: + self.sender.send(msg) + self.log.debug("Message %s sent", msg.id) + else: + self.log.warning("Sender not ready or no credit available") + + def _handle_failed_delivery(self, reason: str): + if self.pending: + msg = self.pending + self.log.warning("Message %s failed for reason: %s", msg.id, reason) + max_retries = self.rconf.radas_sign_timeout_retry_count() + if self.retried < max_retries: + # Schedule retry + self.retried = self.retried + 1 + self.log.info("Scheduling retry %s/%s for message %s", + self.retried, max_retries, msg.id) + # Schedule retry after delay + if self.container: + self.container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) + else: + # Max retries exceeded + self.log.error("Message %s failed after %s retries", msg.id, max_retries) + self.status = "failed" + self.pending = None + else: + self.log.info("Message has been sent successfully, close connection") + self.close() + def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ @@ -291,8 +325,8 @@ def sign_in_radas(repo_url: str, This function will be responsible to do the overall controlling of the whole process, like trigger the send and register the receiver, and control the wait and timeout there. """ - logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," - "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s", + repo_url, requester, sign_key, result_path) request_id = str(uuid.uuid4()) exclude = ignore_patterns if ignore_patterns else [] @@ -305,15 +339,16 @@ def sign_in_radas(repo_url: str, "exclude": exclude } - listener = RadasReceiver(result_path, request_id) - sender = RadasSender(json.dumps(payload)) + sender = RadasSender(json.dumps(payload), radas_config) + container = Container(sender) + container.run() - try: - Container(sender).run() - logger.info("Successfully sent signing request ID: %s", request_id) - Container(listener).run() - finally: - if listener.conn is not None: - listener.conn.close() - if sender.conn is not None: - sender.conn.close() + if not sender.status == "success": + logger.error("Something wrong happened in message sending, see logs") + sys.exit(1) + + listener = RadasReceiver(result_path, request_id, radas_config) + Container(listener).run() + + if listener.conn: + listener.conn.close() diff --git a/pyproject.toml b/pyproject.toml index 43ab9cb4..536df23e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ dependencies = [ "subresource-integrity>=0.2", "jsonschema>=4.9.1", "urllib3>=1.25.10", - "semantic-version>=2.10.0" + "semantic-version>=2.10.0", + "python-qpid-proton>=0.39.0" ] [project.optional-dependencies] diff --git a/requirements-dev.txt b/requirements-dev.txt index f0ed1644..bc38b20f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ -r tests/requirements.txt pyflakes pep8 +tox diff --git a/setup.py b/setup.py index 692b53eb..e2232b75 100755 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "subresource-integrity>=0.2", "jsonschema>=4.9.1", "urllib3>=1.25.10", - "semantic-version>=2.10.0" + "semantic-version>=2.10.0", + "python-qpid-proton>=0.39.0" ], ) diff --git a/tests/test_radas_send_handler.py b/tests/test_radas_send_handler.py deleted file mode 100644 index b37fa05a..00000000 --- a/tests/test_radas_send_handler.py +++ /dev/null @@ -1,98 +0,0 @@ -import tempfile -import os -from unittest import mock -import unittest -from charon.pkgs.radas_signature_handler import sign_in_radas - - -class RadasSignHandlerTest(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - - def tearDown(self) -> None: - super().tearDown() - - def test_sign_in_radas_normal_flow(self): - with tempfile.TemporaryDirectory() as tmpdir: - # Mock configuration - mock_config = mock.MagicMock() - mock_config.is_radas_enabled.return_value = True - mock_radas_config = mock.MagicMock() - mock_config.get_radas_config.return_value = mock_radas_config - - # Mock Container run to avoid real AMQP connection - with mock.patch( - "charon.pkgs.radas_signature_handler.Container") as mock_container, \ - mock.patch( - "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ - mock.patch( - "charon.pkgs.radas_signature_handler.uuid.uuid4", return_value="mocked-uuid"): - - test_result_path = os.path.join(tmpdir, "results") - os.makedirs(test_result_path) - - sign_in_radas( - repo_url="quay.io/test/repo", - requester="test-user", - sign_key="test-key", - result_path=test_result_path, - ignore_patterns=[], - radas_config=mock_radas_config - ) - - # Verify Container.run() was called twice (sender and receiver) - self.assertEqual(mock_container.call_count, 2) - - # Verify request ID propagation - receiver_call = mock_container.call_args_list[1] - self.assertEqual(receiver_call.args[0].request_id, "mocked-uuid") - - def test_sign_in_radas_with_disabled_config(self): - mock_config = mock.MagicMock() - mock_config.is_radas_enabled.return_value = False - - with mock.patch( - "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ - self.assertRaises(SystemExit): - - sign_in_radas( - repo_url="quay.io/test/repo", - requester="test-user", - sign_key="test-key", - result_path="/tmp/results", - ignore_patterns=[], - radas_config=mock.MagicMock() - ) - - def test_sign_in_radas_connection_cleanup(self): - mock_config = mock.MagicMock() - mock_config.is_radas_enabled.return_value = True - mock_radas_config = mock.MagicMock() - - with mock.patch("charon.pkgs.radas_signature_handler.Container") as mock_container, \ - mock.patch("charon.pkgs.radas_signature_handler.get_config", return_value=mock_config): - - mock_sender_conn = mock.MagicMock() - mock_listener_conn = mock.MagicMock() - - def container_side_effect(*args, **kwargs): - if args[0].__class__.__name__ == "RadasReceiver": - args[0].conn = mock_listener_conn - elif args[0].__class__.__name__ == "RadasSender": - args[0].conn = mock_sender_conn - return mock.MagicMock() - - mock_container.side_effect = container_side_effect - - sign_in_radas( - repo_url="quay.io/test/repo", - requester="test-user", - sign_key="test-key", - result_path="/tmp/results", - ignore_patterns=[], - radas_config=mock_radas_config - ) - - # Verify connections are closed - mock_sender_conn.close.assert_called_once() - mock_listener_conn.close.assert_called_once() diff --git a/tests/test_radas_sign_handler.py b/tests/test_radas_sign_generation.py similarity index 99% rename from tests/test_radas_sign_handler.py rename to tests/test_radas_sign_generation.py index b667eaea..33c3d695 100644 --- a/tests/test_radas_sign_handler.py +++ b/tests/test_radas_sign_generation.py @@ -23,7 +23,7 @@ import builtins from unittest import mock from charon.utils.files import overwrite_file -from charon.pkgs.radas_signature_handler import generate_radas_sign +from charon.pkgs.radas_sign import generate_radas_sign logger = logging.getLogger(__name__) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py new file mode 100644 index 00000000..1e75b8fe --- /dev/null +++ b/tests/test_radas_sign_sender.py @@ -0,0 +1,86 @@ +import json +from unittest import mock +import unittest +from charon.pkgs.radas_sign import RadasSender + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_radas_sender(self): + # Mock configuration + mock_radas_config = mock.MagicMock() + mock_radas_config.validate.return_value = True + mock_radas_config.client_ca.return_value = "test-client-ca" + mock_radas_config.client_key.return_value = "test-client-key" + mock_radas_config.client_key_password.return_value = "test-client-key-pass" + mock_radas_config.root_ca.return_value = "test-root-ca" + mock_radas_config.radas_sign_timeout_retry_count.return_value = 5 + + test_payload = { + "request_id": "mock-id", + "requested_by": "test-user", + "type": "mrrc", + "file_reference": "quay.io/test/repo", + "sig_keyname": "test-key", + "exclude": [] + } + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_sign.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_sign.SSLDomain") as ssl_domain, \ + mock.patch("charon.pkgs.radas_sign.Event") as event: + + json_payload = json.dumps(test_payload) + r_sender = RadasSender(json_payload, mock_radas_config) + self.assertEqual(ssl_domain.call_count, 1) + self.assertEqual(r_sender.payload, json_payload) + self.assertIs(r_sender.rconf, mock_radas_config) + self.assertIsNone(r_sender.message) + self.assertIsNone(r_sender.pending) + + # test on_start + mock_sender = mock.MagicMock() + mock_conn = mock.MagicMock() + mock_container.connect.return_value = mock_conn + mock_container.create_sender.return_value = mock_sender + event.container = mock_container + r_sender.on_start(event) + self.assertEqual(mock_container.connect.call_count, 1) + self.assertEqual(mock_container.create_sender.call_count, 1) + + # test on_sendable + mock_sender.credit = 1 + r_sender.on_sendable(event) + self.assertIsNotNone(r_sender.message) + self.assertEqual(mock_sender.send.call_count, 1) + + # test on_accepted + r_sender.on_accepted(event) + self.assertEqual(r_sender.status, "success") + self.assertEqual(r_sender.retried, 0) + self.assertEqual(r_sender.sender.close.call_count, 1) + self.assertEqual(r_sender.container.stop.call_count, 1) + + # test on_rejected + r_sender.on_rejected(event) + self.assertIsNone(r_sender.pending) + self.assertEqual(r_sender.retried, 1) + self.assertEqual(r_sender.container.schedule.call_count, 1) + + # test on_released + r_sender.on_released(event) + self.assertIsNone(r_sender.pending) + self.assertEqual(r_sender.retried, 2) + self.assertEqual(r_sender.container.schedule.call_count, 2) + + # test on_released + r_sender.on_timer_task(event) + self.assertIsNone(r_sender.pending) + self.assertEqual(r_sender.retried, 2) + self.assertEqual(mock_sender.send.call_count, 2) From 5cff6d44fb599266d0d27c2035cfec57b764bff1 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 5 Jun 2025 10:52:15 +0800 Subject: [PATCH 25/64] Refactor: Refactor the RadasReceiver * Support timeout mechanism for Receiver exit --- charon/config.py | 4 + charon/pkgs/radas_sign.py | 113 ++++++++++++++++++-------- tests/test_radas_sign_receiver.py | 130 ++++++++++++++++++++++++++++++ tests/test_radas_sign_sender.py | 2 +- 4 files changed, 214 insertions(+), 35 deletions(-) create mode 100644 tests/test_radas_sign_receiver.py diff --git a/charon/config.py b/charon/config.py index 396e4be3..39e9dc80 100644 --- a/charon/config.py +++ b/charon/config.py @@ -42,6 +42,7 @@ def __init__(self, data: Dict): self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 ) + self.__radas_receiver_timeout: int = int(data.get("radas_receiver_timeout", 1800)) def validate(self) -> bool: if not self.__umb_host: @@ -112,6 +113,9 @@ def radas_sign_timeout_retry_count(self) -> int: def radas_sign_timeout_retry_interval(self) -> int: return self.__radas_sign_timeout_retry_interval + def receiver_timeout(self) -> int: + return self.__radas_receiver_timeout + class CharonConfig(object): """CharonConfig is used to store all configurations for charon diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 355daa77..0ac558b8 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -17,13 +17,14 @@ import logging import json import os -import asyncio import sys +import asyncio import uuid +import time from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import RadasConfig from charon.pkgs.oras_client import OrasClient -from proton import SSLDomain, Message, Event, Sender +from proton import SSLDomain, Message, Event, Sender, Connection from proton.handlers import MessagingHandler from proton.reactor import Container @@ -40,6 +41,8 @@ class RadasReceiver(MessagingHandler): from the cmd flag,should register UmbListener when the client starts request_id (str): Identifier of the request for the signing result + rconf (RadasConfig): + the configurations for the radas messaging system. sign_result_status (str): Result of the signing(success/failed) sign_result_errors (list): @@ -50,10 +53,13 @@ def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id - self.conn = None + self.conn: Optional[Connection] = None + self.message_handled = False self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] self.rconf = rconf + self.start_time = 0.0 + self.timeout_check_delay = 30.0 self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) self.ssl.set_trusted_ca_db(self.rconf.root_ca()) self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) @@ -62,27 +68,58 @@ def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> self.rconf.client_key(), self.rconf.client_key_password() ) + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasReceiver") def on_start(self, event: Event) -> None: - self.conn = event.container.connect( - url=self.rconf.umb_target(), - ssl_domain=self.ssl + umb_target = self.rconf.umb_target() + container = event.container + self.conn = container.connect( + url=umb_target, + ssl_domain=self.ssl, + heartbeat=500 ) - event.container.create_receiver( - self.conn, self.rconf.result_queue(), dynamic=True + receiver = container.create_receiver( + context=self.conn, source=self.rconf.result_queue(), ) - logger.info("Listening on %s, queue: %s", - self.rconf.umb_target(), - self.rconf.result_queue()) + self.log.info("Listening on %s, queue: %s", + umb_target, + receiver.source.address) + self.start_time = time.time() + container.schedule(self.timeout_check_delay, self) + + def on_timer_task(self, event: Event) -> None: + current = time.time() + timeout = self.rconf.receiver_timeout() + idle_time = current - self.start_time + self.log.debug("Checking timeout: passed %s seconds, timeout time %s seconds", + idle_time, timeout) + if idle_time > self.rconf.receiver_timeout(): + self.log.error("The receiver did not receive messages for more than %s seconds," + " and needs to stop receiving and quit.", timeout) + self._close(event) + else: + event.container.schedule(self.timeout_check_delay, self) def on_message(self, event: Event) -> None: + self.log.debug("Got message: %s", event.message.body) self._process_message(event.message.body) + if self.message_handled: + self.log.debug("The signing result is handled.") + self._close(event) - def on_connection_error(self, event: Event) -> None: - logger.error("Received an error event:\n%s", event) + def on_error(self, event: Event) -> None: + self.log.error("Received an error event:\n%s", event.message.body) def on_disconnected(self, event: Event) -> None: - logger.error("Disconnected from AMQP broker.") + self.log.info("Disconnected from AMQP broker: %s", + event.connection.connected_address) + + def _close(self, event: Event) -> None: + if event: + if event.connection: + event.connection.close() + if event.container: + event.container.stop() def _process_message(self, msg: Any) -> None: """ @@ -93,32 +130,37 @@ def _process_message(self, msg: Any) -> None: msg_dict = json.loads(msg) msg_request_id = msg_dict.get("request_id") if msg_request_id != self.request_id: - logger.info( + self.log.info( "Message request_id %s does not match the request_id %s from sender, ignoring", msg_request_id, self.request_id, ) return - logger.info( + self.message_handled = True + self.log.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) self.sign_result_status = msg_dict.get("signing_status") self.sign_result_errors = msg_dict.get("errors", []) - result_reference_url = msg_dict.get("result_reference") - if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return + if self.sign_result_status == "success": + result_reference_url = msg_dict.get("result_reference") + if not result_reference_url: + self.log.warning("Not found result_reference in message，ignore.") + return - logger.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) - sign_result_parent_dir = os.path.dirname(self.sign_result_loc) - os.makedirs(sign_result_parent_dir, exist_ok=True) + self.log.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) - oras_client = OrasClient() - files = oras_client.pull( - result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc - ) - logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) + else: + self.log.error("The signing result received with failed status. Errors: %s", + self.sign_result_errors) class RadasSender(MessagingHandler): @@ -141,7 +183,6 @@ def __init__(self, payload: Any, rconf: RadasConfig): self.message: Optional[Message] = None self.container: Optional[Container] = None self.sender: Optional[Sender] = None - self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) self.ssl.set_trusted_ca_db(self.rconf.root_ca()) self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) @@ -150,6 +191,7 @@ def __init__(self, payload: Any, rconf: RadasConfig): self.rconf.client_key(), self.rconf.client_key_password() ) + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") def on_start(self, event): self.container = event.container @@ -329,7 +371,6 @@ def sign_in_radas(repo_url: str, repo_url, requester, sign_key, result_path) request_id = str(uuid.uuid4()) exclude = ignore_patterns if ignore_patterns else [] - payload = { "request_id": request_id, "requested_by": requester, @@ -347,8 +388,12 @@ def sign_in_radas(repo_url: str, logger.error("Something wrong happened in message sending, see logs") sys.exit(1) - listener = RadasReceiver(result_path, request_id, radas_config) - Container(listener).run() + # request_id = "some-request-id-1" # for test purpose + receiver = RadasReceiver(result_path, request_id, radas_config) + Container(receiver).run() - if listener.conn: - listener.conn.close() + status = receiver.sign_result_status + if status != "success": + logger.error("The signing result is processed with errors: %s", + receiver.sign_result_errors) + sys.exit(1) diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py new file mode 100644 index 00000000..c75a8e64 --- /dev/null +++ b/tests/test_radas_sign_receiver.py @@ -0,0 +1,130 @@ +from unittest import mock +import unittest +import tempfile +import time +import json +from charon.pkgs.radas_sign import RadasReceiver + + +class RadasSignReceiverTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def reset_receiver(self, r_receiver: RadasReceiver) -> None: + r_receiver.message_handled = False + r_receiver.sign_result_errors = [] + r_receiver.sign_result_status = None + + def test_radas_receiver(self): + # Mock configuration + mock_radas_config = mock.MagicMock() + mock_radas_config.validate.return_value = True + mock_radas_config.client_ca.return_value = "test-client-ca" + mock_radas_config.client_key.return_value = "test-client-key" + mock_radas_config.client_key_password.return_value = "test-client-key-pass" + mock_radas_config.root_ca.return_value = "test-root-ca" + mock_radas_config.receiver_timeout.return_value = 60 + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_sign.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_sign.SSLDomain") as ssl_domain, \ + mock.patch("charon.pkgs.radas_sign.OrasClient") as oras_client, \ + mock.patch("charon.pkgs.radas_sign.Event") as event: + test_result_path = tempfile.mkdtemp() + test_request_id = "test-request-id" + r_receiver = RadasReceiver(test_result_path, test_request_id, mock_radas_config) + self.assertEqual(ssl_domain.call_count, 1) + self.assertEqual(r_receiver.sign_result_loc, test_result_path) + self.assertEqual(r_receiver.request_id, test_request_id) + + # prepare mock + mock_receiver = mock.MagicMock() + mock_conn = mock.MagicMock() + mock_container.connect.return_value = mock_conn + mock_container.create_receiver.return_value = mock_receiver + event.container = mock_container + event.message = mock.MagicMock() + event.connection = mock.MagicMock() + + # test on_start + r_receiver.on_start(event) + self.assertEqual(mock_container.connect.call_count, 1) + self.assertEqual(mock_container.create_receiver.call_count, 1) + self.assertTrue(r_receiver.start_time > 0.0) + self.assertTrue(r_receiver.start_time < time.time()) + self.assertEqual(mock_container.schedule.call_count, 1) + + # test on_message: unmatched case + test_ummatch_result = { + "request_id": "test-request-id-no-match", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } + event.message.body = json.dumps(test_ummatch_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 0) + self.assertEqual(mock_container.stop.call_count, 0) + self.assertFalse(r_receiver.message_handled) + self.assertIsNone(r_receiver.sign_result_status) + self.assertEqual(r_receiver.sign_result_errors, []) + self.assertEqual(oras_client.call_count, 0) + + # test on_message: matched case with failed status + self.reset_receiver(r_receiver) + test_failed_result = { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "failed", + "errors": ["error1", "error2"] + } + event.message.body = json.dumps(test_failed_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 1) + self.assertEqual(mock_container.stop.call_count, 1) + self.assertTrue(r_receiver.message_handled) + self.assertEqual(r_receiver.sign_result_status, "failed") + self.assertEqual(r_receiver.sign_result_errors, ["error1", "error2"]) + self.assertEqual(oras_client.call_count, 0) + + # test on_message: matched case with success status + self.reset_receiver(r_receiver) + test_success_result = { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } + event.message.body = json.dumps(test_success_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 2) + self.assertEqual(mock_container.stop.call_count, 2) + self.assertTrue(r_receiver.message_handled) + self.assertEqual(r_receiver.sign_result_status, "success") + self.assertEqual(r_receiver.sign_result_errors, []) + self.assertEqual(oras_client.call_count, 1) + oras_client_call = oras_client.return_value + self.assertEqual(oras_client_call.pull.call_count, 1) + + # test on_timer_task: not timeout + r_receiver.on_timer_task(event) + self.assertEqual(event.connection.close.call_count, 2) + self.assertEqual(mock_container.stop.call_count, 2) + self.assertEqual(mock_container.schedule.call_count, 2) + + # test on_timer_task: timeout + mock_radas_config.receiver_timeout.return_value = 0 + r_receiver.on_timer_task(event) + self.assertEqual(event.connection.close.call_count, 3) + self.assertEqual(mock_container.stop.call_count, 3) + self.assertEqual(mock_container.schedule.call_count, 2) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py index 1e75b8fe..602d7de6 100644 --- a/tests/test_radas_sign_sender.py +++ b/tests/test_radas_sign_sender.py @@ -4,7 +4,7 @@ from charon.pkgs.radas_sign import RadasSender -class RadasSignHandlerTest(unittest.TestCase): +class RadasSignSenderTest(unittest.TestCase): def setUp(self) -> None: super().setUp() From 6340f91df49a70fdc98372429da57d3230eb80c9 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 5 Jun 2025 16:07:21 +0800 Subject: [PATCH 26/64] Some chore changes: * Add ssl enable condition in RadasConfig * Mark some fields as "private" for RadasSender and RadasReceiver --- charon/cmd/cmd_sign.py | 2 - charon/config.py | 3 + charon/pkgs/radas_sign.py | 126 ++++++++++++++++-------------- tests/test_radas_sign_receiver.py | 12 +-- tests/test_radas_sign_sender.py | 28 +++---- 5 files changed, 89 insertions(+), 82 deletions(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index a83a3e3f..f17fe028 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -109,8 +109,6 @@ def sign( try: current = datetime.datetime.now().strftime("%Y%m%d%I%M") _decide_mode("radas_sign", current, is_quiet=quiet, is_debug=debug) - if dryrun: - logger.info("Running in dry-run mode, no files will signed.") conf = get_config(config) if not conf: logger.error("The charon configuration is not valid!") diff --git a/charon/config.py b/charon/config.py index 39e9dc80..a99c1436 100644 --- a/charon/config.py +++ b/charon/config.py @@ -102,6 +102,9 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca.strip() + def ssl_enabled(self) -> bool: + return bool(self.__client_ca and self.__client_key and self.__root_ca) + def quay_radas_registry_config(self) -> Optional[str]: if self.__quay_radas_registry_config: return self.__quay_radas_registry_config.strip() diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 0ac558b8..d5c50cca 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -53,44 +53,46 @@ def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id - self.conn: Optional[Connection] = None - self.message_handled = False self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] self.rconf = rconf - self.start_time = 0.0 - self.timeout_check_delay = 30.0 - self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) - self.ssl.set_trusted_ca_db(self.rconf.root_ca()) - self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) - self.ssl.set_credentials( - self.rconf.client_ca(), - self.rconf.client_key(), - self.rconf.client_key_password() - ) + self._conn: Optional[Connection] = None + self._message_handled = False + self._start_time = 0.0 + self._timeout_check_delay = 30.0 + self._ssl: Optional[SSLDomain] = None + if rconf.ssl_enabled(): + self._ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self._ssl.set_trusted_ca_db(self.rconf.root_ca()) + self._ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self._ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() + ) self.log = logging.getLogger("charon.pkgs.radas_sign.RadasReceiver") def on_start(self, event: Event) -> None: umb_target = self.rconf.umb_target() container = event.container - self.conn = container.connect( + self._conn = container.connect( url=umb_target, - ssl_domain=self.ssl, + ssl_domain=self._ssl, heartbeat=500 ) receiver = container.create_receiver( - context=self.conn, source=self.rconf.result_queue(), + context=self._conn, source=self.rconf.result_queue(), ) self.log.info("Listening on %s, queue: %s", umb_target, receiver.source.address) - self.start_time = time.time() - container.schedule(self.timeout_check_delay, self) + self._start_time = time.time() + container.schedule(self._timeout_check_delay, self) def on_timer_task(self, event: Event) -> None: current = time.time() timeout = self.rconf.receiver_timeout() - idle_time = current - self.start_time + idle_time = current - self._start_time self.log.debug("Checking timeout: passed %s seconds, timeout time %s seconds", idle_time, timeout) if idle_time > self.rconf.receiver_timeout(): @@ -98,12 +100,12 @@ def on_timer_task(self, event: Event) -> None: " and needs to stop receiving and quit.", timeout) self._close(event) else: - event.container.schedule(self.timeout_check_delay, self) + event.container.schedule(self._timeout_check_delay, self) def on_message(self, event: Event) -> None: self.log.debug("Got message: %s", event.message.body) self._process_message(event.message.body) - if self.message_handled: + if self._message_handled: self.log.debug("The signing result is handled.") self._close(event) @@ -137,7 +139,7 @@ def _process_message(self, msg: Any) -> None: ) return - self.message_handled = True + self._message_handled = True self.log.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) @@ -171,44 +173,48 @@ class RadasSender(MessagingHandler): this value construct from the cmd flag rconf (RadasConfig): the configurations for the radas messaging system. + status (str): tell if status for message sending, only "success" + means the message is sent successfully. """ def __init__(self, payload: Any, rconf: RadasConfig): super(RadasSender, self).__init__() self.payload = payload self.rconf = rconf - self.message_sent = False # Flag to track if message was sent self.status: Optional[str] = None - self.retried = 0 - self.pending: Optional[Message] = None - self.message: Optional[Message] = None - self.container: Optional[Container] = None - self.sender: Optional[Sender] = None - self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) - self.ssl.set_trusted_ca_db(self.rconf.root_ca()) - self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) - self.ssl.set_credentials( - self.rconf.client_ca(), - self.rconf.client_key(), - self.rconf.client_key_password() - ) + self._message_sent = False # Flag to track if message was sent + self._retried = 0 + self._pending: Optional[Message] = None + self._message: Optional[Message] = None + self._container: Optional[Container] = None + self._sender: Optional[Sender] = None + self._ssl: Optional[SSLDomain] = None + if self.rconf.ssl_enabled(): + self._ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self._ssl.set_trusted_ca_db(self.rconf.root_ca()) + self._ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self._ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() + ) self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") def on_start(self, event): - self.container = event.container - conn = self.container.connect( + self._container = event.container + conn = self._container.connect( url=self.rconf.umb_target(), - ssl_domain=self.ssl + ssl_domain=self._ssl ) if conn: - self.sender = self.container.create_sender(conn, self.rconf.request_queue()) + self._sender = self._container.create_sender(conn, self.rconf.request_queue()) def on_sendable(self, event): - if not self.message_sent: + if not self._message_sent: msg = Message(body=self.payload, durable=True) self.log.debug("Sending message: %s to %s", msg.id, event.sender.target.address) self._send_msg(msg) - self.message = msg - self.message_sent = True + self._message = msg + self._message_sent = True def on_error(self, event): self.log.error("Error happened during message sending, reason %s", @@ -216,11 +222,11 @@ def on_error(self, event): self.status = "failed" def on_rejected(self, event): - self.pending = self.message + self._pending = self._message self._handle_failed_delivery("Rejected") def on_released(self, event): - self.pending = self.message + self._pending = self._message self._handle_failed_delivery("Released") def on_accepted(self, event): @@ -229,42 +235,42 @@ def on_accepted(self, event): self.close() # Close connection after confirmation def on_timer_task(self, event): - message_to_retry = self.message + message_to_retry = self._message self._send_msg(message_to_retry) - self.pending = None + self._pending = None def close(self): self.log.info("Message has been sent successfully, close connection") - if self.sender: - self.sender.close() - if self.container: - self.container.stop() + if self._sender: + self._sender.close() + if self._container: + self._container.stop() def _send_msg(self, msg: Message): - if self.sender and self.sender.credit > 0: - self.sender.send(msg) + if self._sender and self._sender.credit > 0: + self._sender.send(msg) self.log.debug("Message %s sent", msg.id) else: self.log.warning("Sender not ready or no credit available") def _handle_failed_delivery(self, reason: str): - if self.pending: - msg = self.pending + if self._pending: + msg = self._pending self.log.warning("Message %s failed for reason: %s", msg.id, reason) max_retries = self.rconf.radas_sign_timeout_retry_count() - if self.retried < max_retries: + if self._retried < max_retries: # Schedule retry - self.retried = self.retried + 1 + self._retried = self._retried + 1 self.log.info("Scheduling retry %s/%s for message %s", - self.retried, max_retries, msg.id) + self._retried, max_retries, msg.id) # Schedule retry after delay - if self.container: - self.container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) + if self._container: + self._container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) else: # Max retries exceeded self.log.error("Message %s failed after %s retries", msg.id, max_retries) self.status = "failed" - self.pending = None + self._pending = None else: self.log.info("Message has been sent successfully, close connection") self.close() diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py index c75a8e64..e80f0435 100644 --- a/tests/test_radas_sign_receiver.py +++ b/tests/test_radas_sign_receiver.py @@ -14,7 +14,7 @@ def tearDown(self) -> None: super().tearDown() def reset_receiver(self, r_receiver: RadasReceiver) -> None: - r_receiver.message_handled = False + r_receiver._message_handled = False r_receiver.sign_result_errors = [] r_receiver.sign_result_status = None @@ -54,8 +54,8 @@ def test_radas_receiver(self): r_receiver.on_start(event) self.assertEqual(mock_container.connect.call_count, 1) self.assertEqual(mock_container.create_receiver.call_count, 1) - self.assertTrue(r_receiver.start_time > 0.0) - self.assertTrue(r_receiver.start_time < time.time()) + self.assertTrue(r_receiver._start_time > 0.0) + self.assertTrue(r_receiver._start_time < time.time()) self.assertEqual(mock_container.schedule.call_count, 1) # test on_message: unmatched case @@ -71,7 +71,7 @@ def test_radas_receiver(self): r_receiver.on_message(event) self.assertEqual(event.connection.close.call_count, 0) self.assertEqual(mock_container.stop.call_count, 0) - self.assertFalse(r_receiver.message_handled) + self.assertFalse(r_receiver._message_handled) self.assertIsNone(r_receiver.sign_result_status) self.assertEqual(r_receiver.sign_result_errors, []) self.assertEqual(oras_client.call_count, 0) @@ -90,7 +90,7 @@ def test_radas_receiver(self): r_receiver.on_message(event) self.assertEqual(event.connection.close.call_count, 1) self.assertEqual(mock_container.stop.call_count, 1) - self.assertTrue(r_receiver.message_handled) + self.assertTrue(r_receiver._message_handled) self.assertEqual(r_receiver.sign_result_status, "failed") self.assertEqual(r_receiver.sign_result_errors, ["error1", "error2"]) self.assertEqual(oras_client.call_count, 0) @@ -109,7 +109,7 @@ def test_radas_receiver(self): r_receiver.on_message(event) self.assertEqual(event.connection.close.call_count, 2) self.assertEqual(mock_container.stop.call_count, 2) - self.assertTrue(r_receiver.message_handled) + self.assertTrue(r_receiver._message_handled) self.assertEqual(r_receiver.sign_result_status, "success") self.assertEqual(r_receiver.sign_result_errors, []) self.assertEqual(oras_client.call_count, 1) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py index 602d7de6..c1c1fee3 100644 --- a/tests/test_radas_sign_sender.py +++ b/tests/test_radas_sign_sender.py @@ -41,8 +41,8 @@ def test_radas_sender(self): self.assertEqual(ssl_domain.call_count, 1) self.assertEqual(r_sender.payload, json_payload) self.assertIs(r_sender.rconf, mock_radas_config) - self.assertIsNone(r_sender.message) - self.assertIsNone(r_sender.pending) + self.assertIsNone(r_sender._message) + self.assertIsNone(r_sender._pending) # test on_start mock_sender = mock.MagicMock() @@ -57,30 +57,30 @@ def test_radas_sender(self): # test on_sendable mock_sender.credit = 1 r_sender.on_sendable(event) - self.assertIsNotNone(r_sender.message) + self.assertIsNotNone(r_sender._message) self.assertEqual(mock_sender.send.call_count, 1) # test on_accepted r_sender.on_accepted(event) self.assertEqual(r_sender.status, "success") - self.assertEqual(r_sender.retried, 0) - self.assertEqual(r_sender.sender.close.call_count, 1) - self.assertEqual(r_sender.container.stop.call_count, 1) + self.assertEqual(r_sender._retried, 0) + self.assertEqual(r_sender._sender.close.call_count, 1) + self.assertEqual(r_sender._container.stop.call_count, 1) # test on_rejected r_sender.on_rejected(event) - self.assertIsNone(r_sender.pending) - self.assertEqual(r_sender.retried, 1) - self.assertEqual(r_sender.container.schedule.call_count, 1) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 1) + self.assertEqual(r_sender._container.schedule.call_count, 1) # test on_released r_sender.on_released(event) - self.assertIsNone(r_sender.pending) - self.assertEqual(r_sender.retried, 2) - self.assertEqual(r_sender.container.schedule.call_count, 2) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 2) + self.assertEqual(r_sender._container.schedule.call_count, 2) # test on_released r_sender.on_timer_task(event) - self.assertIsNone(r_sender.pending) - self.assertEqual(r_sender.retried, 2) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 2) self.assertEqual(mock_sender.send.call_count, 2) From 3278f224be42d4a9708a3749ecc842111f68a8e0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 5 Jun 2025 20:29:27 +0800 Subject: [PATCH 27/64] RADAS: change the request_queue to request_channel in config As topic is also a valid destination to send, use queue is not correct --- charon/config.py | 8 ++++---- charon/pkgs/radas_sign.py | 2 +- tests/test_config_radas.py | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/charon/config.py b/charon/config.py index a99c1436..d98bb995 100644 --- a/charon/config.py +++ b/charon/config.py @@ -30,7 +30,7 @@ def __init__(self, data: Dict): self.__umb_host: str = data.get("umb_host", None) self.__umb_host_port: str = data.get("umb_host_port", "5671") self.__result_queue: str = data.get("result_queue", None) - self.__request_queue: str = data.get("request_queue", None) + self.__request_chan: str = data.get("request_channel", None) self.__client_ca: str = data.get("client_ca", None) self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) @@ -51,7 +51,7 @@ def validate(self) -> bool: if not self.__result_queue: logger.error("Missing the queue setting to receive signing result in UMB!") return False - if not self.__request_queue: + if not self.__request_chan: logger.error("Missing the queue setting to send signing request in UMB!") return False if self.__client_ca and not os.access(self.__client_ca, os.R_OK): @@ -81,8 +81,8 @@ def umb_target(self) -> str: def result_queue(self) -> str: return self.__result_queue.strip() - def request_queue(self) -> str: - return self.__request_queue.strip() + def request_channel(self) -> str: + return self.__request_chan.strip() def client_ca(self) -> str: return self.__client_ca.strip() diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index d5c50cca..7ac4e3dc 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -206,7 +206,7 @@ def on_start(self, event): ssl_domain=self._ssl ) if conn: - self._sender = self._container.create_sender(conn, self.rconf.request_queue()) + self._sender = self._container.create_sender(conn, self.rconf.request_channel()) def on_sendable(self, event): if not self._message_sent: diff --git a/tests/test_config_radas.py b/tests/test_config_radas.py index 152dc1c2..a6c7d5a4 100644 --- a/tests/test_config_radas.py +++ b/tests/test_config_radas.py @@ -36,7 +36,7 @@ def test_full_radas_config(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -59,7 +59,7 @@ def test_missing_umb_host(self): radas_settings = """ radas: result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -79,7 +79,7 @@ def test_missing_result_queue(self): radas_settings = """ radas: umb_host: test.umb.api.com - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -120,7 +120,7 @@ def test_unaccessible_client_ca(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -142,7 +142,7 @@ def test_unaccessible_client_key(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -164,7 +164,7 @@ def test_unaccessible_client_password_file(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -186,7 +186,7 @@ def test_unaccessible_root_ca(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} From 3e7187501b1bdfcad0cf29037e8ee0eacf2a6141 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 6 Jun 2025 18:22:27 +0800 Subject: [PATCH 28/64] Add missing oras deps in project.toml and setup.py --- pyproject.toml | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 536df23e..ea54ee99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", + "oras>=0.2.31", "python-qpid-proton>=0.39.0" ] diff --git a/setup.py b/setup.py index e2232b75..8628953c 100755 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", + "oras>=0.2.31", "python-qpid-proton>=0.39.0" ], ) From a38e565b0dcb89aa90bc0c4723401548055d6357 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 9 Jun 2025 15:17:37 +0800 Subject: [PATCH 29/64] RADAS: update Containerfile for radas support * Update base image to new version of ubi8-minimal * Add gcc, openssl-devel as dependencies for build * Use python12-devel instead of python12 because qpid-proton --- image/Containerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/image/Containerfile b/image/Containerfile index f411bdb0..31b31060 100644 --- a/image/Containerfile +++ b/image/Containerfile @@ -19,16 +19,16 @@ # 4. Start using uploader # charon upload/delete from /home/charon/upload/... ### -FROM registry.access.redhat.com/ubi8-minimal:8.10-1052 as builder +FROM registry.access.redhat.com/ubi8-minimal:8.10-1295 as builder ARG GIT_BRANCH=main -RUN microdnf install -y git-core python3.12 python3.12-pip && microdnf clean all +RUN microdnf install -y git-core python3.12-devel python3.12-pip gcc openssl-devel && microdnf clean all RUN git clone -b ${GIT_BRANCH} --depth 1 https://github.com/Commonjava/charon.git RUN pip3 install --no-cache-dir --upgrade pip RUN pip3 wheel ./charon -FROM registry.access.redhat.com/ubi8-minimal:8.10-1052 +FROM registry.access.redhat.com/ubi8-minimal:8.10-1295 ARG USER=charon ARG UID=10000 @@ -38,7 +38,7 @@ WORKDIR ${HOME_DIR} USER root -RUN microdnf install -y python3.12 python3.12-pip shadow-utils && microdnf clean all +RUN microdnf install -y python3.12-devel python3.12-pip shadow-utils gcc openssl-devel && microdnf clean all RUN useradd -d ${HOME_DIR} -u ${UID} -g 0 -m -s /bin/bash ${USER} \ && chown ${USER}:0 ${HOME_DIR} \ && chmod -R g+rwx ${HOME_DIR} \ From 83e8bca7c6dc04dd6389714f03c08353eb6e038d Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 9 Jun 2025 20:46:29 +0800 Subject: [PATCH 30/64] RADAS: add default ignore patterns for signing --- charon/cmd/cmd_sign.py | 6 ++++-- charon/constants.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index f17fe028..9c901493 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -18,6 +18,7 @@ from charon.config import get_config from charon.pkgs.radas_sign import sign_in_radas from charon.cmd.internal import _decide_mode +from charon.constants import DEFAULT_RADAS_SIGN_IGNORES from click import command, option, argument @@ -98,8 +99,7 @@ def sign( ignore_patterns: List[str] = None, config: str = None, debug=False, - quiet=False, - dryrun=False + quiet=False ): """Do signing against files in the repo zip in repo_url through radas service. The repo_url points to the maven zip repository @@ -119,8 +119,10 @@ def sign( sys.exit(1) # All ignore files in global config should also be ignored in signing. ig_patterns = conf.get_ignore_patterns() + ig_patterns.extend(DEFAULT_RADAS_SIGN_IGNORES) if ignore_patterns: ig_patterns.extend(ignore_patterns) + ig_patterns = list(set(ig_patterns)) args = { "repo_url": repo_url, "requester": requester, diff --git a/charon/constants.py b/charon/constants.py index 35ea560a..e8056dc0 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -177,3 +177,8 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 + +DEFAULT_RADAS_SIGN_IGNORES = [ + r".*\.md5$", r".*\.sha1$", r".*\.sha128$", r".*\.sha256$", + r".*\.sha512$", r".*\.asc$" +] From 821599bfff1c5080266654066dd3a1fe0146a0e0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 11 Jun 2025 10:33:03 +0800 Subject: [PATCH 31/64] Update project version to 1.4.0 --- charon.spec | 2 +- pyproject.toml | 4 ++-- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charon.spec b/charon.spec index 147d9885..7fd84c7f 100644 --- a/charon.spec +++ b/charon.spec @@ -1,7 +1,7 @@ %global owner Commonjava %global modulename charon -%global charon_version 1.3.3 +%global charon_version 1.4.0 %global sdist_tar_name %{modulename}-%{charon_version} %global python3_pkgversion 3 diff --git a/pyproject.toml b/pyproject.toml index ea54ee99..56148155 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["setuptools", "setuptools-scm"] [project] name = "charon" -version = "1.3.3" +version = "1.4.0" authors = [ {name = "RedHat EXD SPMM"}, ] @@ -107,4 +107,4 @@ ignore = [ ] per-file-ignores = [ "tests/*:D101,D102,D103", # missing docstring in public class, method, function -] \ No newline at end of file +] diff --git a/setup.py b/setup.py index 8628953c..ee5eac4b 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ """ from setuptools import setup, find_packages -version = "1.3.3" +version = "1.4.0" long_description = """ This charon is a tool to synchronize several types of From d0d8d120cc7bbfc13c0cbfbf3bf258a9a48f645f Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 11 Jun 2025 20:54:10 +0800 Subject: [PATCH 32/64] RADAS: use result file directly instead of folder in maven upload --- charon/cmd/cmd_upload.py | 4 ++-- charon/pkgs/maven.py | 6 +++--- charon/pkgs/radas_sign.py | 22 ++++------------------ tests/test_radas_sign_generation.py | 21 ++++----------------- 4 files changed, 13 insertions(+), 40 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index d56a644d..2ad294f5 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -161,7 +161,7 @@ def upload( debug=False, quiet=False, dryrun=False, - sign_result_loc="/tmp/sign" + sign_result_file=None, ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -233,7 +233,7 @@ def upload( dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, config=config, - sign_result_loc=sign_result_loc + sign_result_file=sign_result_file ) if not succeeded: sys.exit(1) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 2f525ce3..5ccee694 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -276,7 +276,7 @@ def handle_maven_uploading( dry_run=False, manifest_bucket_name=None, config=None, - sign_result_loc="/tmp/sign" + sign_result_file=None ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -416,10 +416,10 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_enabled(): + if conf.is_radas_enabled() and sign_result_file and os.path.isfile(sign_result_file): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( - top_level=top_level, sign_result_loc=sign_result_loc + top_level=top_level, sign_result_file=sign_result_file ) if not _generated_signs: logger.error( diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 7ac4e3dc..cd229425 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -276,31 +276,17 @@ def _handle_failed_delivery(self, reason: str): self.close() -def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: +def generate_radas_sign(top_level: str, sign_result_file: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ - if not os.path.isdir(sign_result_loc): - logger.error("Sign result loc dir does not exist: %s", sign_result_loc) - return [], [] - - files = [ - os.path.join(sign_result_loc, f) - for f in os.listdir(sign_result_loc) - if os.path.isfile(os.path.join(sign_result_loc, f)) - ] - - if not files: - return [], [] - - if len(files) > 1: - logger.error("Multiple files found in %s. Expected only one file.", sign_result_loc) + if not sign_result_file or not os.path.isfile(sign_result_file): + logger.error("Sign result file does not exist: %s", sign_result_file) return [], [] # should only have the single sign result json file from the radas registry - json_file_path = files[0] try: - with open(json_file_path, "r") as f: + with open(sign_result_file, "r") as f: data = json.load(f) except Exception as e: logger.error("Failed to read or parse the JSON file: %s", e) diff --git a/tests/test_radas_sign_generation.py b/tests/test_radas_sign_generation.py index 33c3d695..ccc448a2 100644 --- a/tests/test_radas_sign_generation.py +++ b/tests/test_radas_sign_generation.py @@ -39,7 +39,7 @@ def tearDown(self) -> None: def test_multi_sign_files_generation(self): self.__prepare_artifacts() - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(failed, []) expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") @@ -55,7 +55,7 @@ def test_multi_sign_files_generation(self): self.assertIn("signature2@hash", content2) def test_sign_files_generation_with_missing_artifacts(self): - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(failed, []) expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") @@ -82,7 +82,7 @@ def side_effect(path, *args, **kwargs): raise IOError("mock write error") return real_open(path, *args, **kwargs) mock_open.side_effect = side_effect - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(len(failed), 1) self.assertNotIn(expected_asc1, generated) @@ -93,20 +93,7 @@ def test_sign_files_generation_with_missing_result(self): # simulate missing pull result by removing the sign result file loc shutil.rmtree(self.__sign_result_loc) - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) - self.assertEqual(failed, []) - expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") - expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") - self.assertEqual(generated, []) - self.assertFalse(os.path.exists(expected_asc1)) - self.assertFalse(os.path.exists(expected_asc2)) - - def test_sign_files_generation_with_not_single_results(self): - self.__prepare_artifacts() - another_result_file = os.path.join(self.__sign_result_loc, "result2.json") - overwrite_file(another_result_file, "test_json") - - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(failed, []) expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") From 59a6290e9fb80ae8ac635a5de89189067c385ae6 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 12 Jun 2025 14:00:45 +0800 Subject: [PATCH 33/64] RADAS: Added some log --- charon/cmd/cmd_sign.py | 1 - charon/pkgs/radas_sign.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index 9c901493..49b2bf22 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -131,7 +131,6 @@ def sign( "ignore_patterns": ig_patterns, "radas_config": radas_conf } - logger.debug("params: %s", args) sign_in_radas(**args) # type: ignore except Exception: print(traceback.format_exc()) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index cd229425..5cd87fd1 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -201,10 +201,12 @@ def __init__(self, payload: Any, rconf: RadasConfig): def on_start(self, event): self._container = event.container + self.log.debug("Start creating connection for sender") conn = self._container.connect( url=self.rconf.umb_target(), ssl_domain=self._ssl ) + self.log.debug("Connection to %s is created.", conn.hostname) if conn: self._sender = self._container.create_sender(conn, self.rconf.request_channel()) From 539b79db29b1d615943840fee09a459760572a78 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 12 Jun 2025 15:12:01 +0800 Subject: [PATCH 34/64] RADAS: adjust some logging --- charon/config.py | 5 ++++- charon/pkgs/radas_sign.py | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/charon/config.py b/charon/config.py index d98bb995..65f23020 100644 --- a/charon/config.py +++ b/charon/config.py @@ -76,7 +76,10 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" + if self.ssl_enabled(): + return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" + else: + return f"amqp://{self.__umb_host.strip()}:{self.__umb_host_port}" def result_queue(self) -> str: return self.__result_queue.strip() diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 5cd87fd1..3ca817cd 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -201,19 +201,25 @@ def __init__(self, payload: Any, rconf: RadasConfig): def on_start(self, event): self._container = event.container - self.log.debug("Start creating connection for sender") + self.log.debug("Start creating connection for sender to %s", self.rconf.umb_target()) conn = self._container.connect( url=self.rconf.umb_target(), - ssl_domain=self._ssl + ssl_domain=self._ssl, + heartbeat=500 ) - self.log.debug("Connection to %s is created.", conn.hostname) if conn: + self.log.debug("Start creating sender") self._sender = self._container.create_sender(conn, self.rconf.request_channel()) + self.log.debug("Sender created. Remote address: %s", self._sender.target.address) + + def on_connection_opened(self, event): + conn = event.connection + self.log.debug("Connection to %s is created.", conn.hostname) def on_sendable(self, event): if not self._message_sent: msg = Message(body=self.payload, durable=True) - self.log.debug("Sending message: %s to %s", msg.id, event.sender.target.address) + self.log.debug("Sending message: %s to %s", msg.body, event.sender.target.address) self._send_msg(msg) self._message = msg self._message_sent = True @@ -232,7 +238,7 @@ def on_released(self, event): self._handle_failed_delivery("Released") def on_accepted(self, event): - self.log.info("Message accepted by receiver: %s", event.delivery) + self.log.info("Message accepted by receiver: %s", event.delivery.link.target.address) self.status = "success" self.close() # Close connection after confirmation @@ -251,26 +257,26 @@ def close(self): def _send_msg(self, msg: Message): if self._sender and self._sender.credit > 0: self._sender.send(msg) - self.log.debug("Message %s sent", msg.id) + self.log.debug("Message %s sent", msg.body) else: self.log.warning("Sender not ready or no credit available") def _handle_failed_delivery(self, reason: str): if self._pending: msg = self._pending - self.log.warning("Message %s failed for reason: %s", msg.id, reason) + self.log.warning("Message %s failed for reason: %s", msg.body, reason) max_retries = self.rconf.radas_sign_timeout_retry_count() if self._retried < max_retries: # Schedule retry self._retried = self._retried + 1 self.log.info("Scheduling retry %s/%s for message %s", - self._retried, max_retries, msg.id) + self._retried, max_retries, msg.body) # Schedule retry after delay if self._container: self._container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) else: # Max retries exceeded - self.log.error("Message %s failed after %s retries", msg.id, max_retries) + self.log.error("Message %s failed after %s retries", msg.body, max_retries) self.status = "failed" self._pending = None else: From 8f4785599d93b25eb7c674802e5a7fafd8d997fd Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 13 Jun 2025 08:15:19 +0800 Subject: [PATCH 35/64] RADAS: fix radas response format --- charon/pkgs/radas_sign.py | 16 ++++++--- tests/test_radas_sign_receiver.py | 57 +++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 3ca817cd..bf63ff64 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -130,7 +130,15 @@ def _process_message(self, msg: Any) -> None: msg: The message body received """ msg_dict = json.loads(msg) - msg_request_id = msg_dict.get("request_id") + radas_response = msg_dict.get("msg") + if not radas_response: + self.log.info( + "Message %s is not valid, ignoring", + msg_dict + ) + return + + msg_request_id = radas_response.get("request_id") if msg_request_id != self.request_id: self.log.info( "Message request_id %s does not match the request_id %s from sender, ignoring", @@ -143,10 +151,10 @@ def _process_message(self, msg: Any) -> None: self.log.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) - self.sign_result_status = msg_dict.get("signing_status") - self.sign_result_errors = msg_dict.get("errors", []) + self.sign_result_status = radas_response.get("signing_status") + self.sign_result_errors = radas_response.get("errors", []) if self.sign_result_status == "success": - result_reference_url = msg_dict.get("result_reference") + result_reference_url = radas_response.get("result_reference") if not result_reference_url: self.log.warning("Not found result_reference in message，ignore.") return diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py index e80f0435..1090c61b 100644 --- a/tests/test_radas_sign_receiver.py +++ b/tests/test_radas_sign_receiver.py @@ -60,12 +60,19 @@ def test_radas_receiver(self): # test on_message: unmatched case test_ummatch_result = { - "request_id": "test-request-id-no-match", - "file_reference": "quay.io/example/test-repo", - "result_reference": "quay.io/example-sign/sign-repo", - "sig_keyname": "testkey", - "signing_status": "success", - "errors": [] + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id-no-match", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } } event.message.body = json.dumps(test_ummatch_result) r_receiver.on_message(event) @@ -79,12 +86,19 @@ def test_radas_receiver(self): # test on_message: matched case with failed status self.reset_receiver(r_receiver) test_failed_result = { - "request_id": "test-request-id", - "file_reference": "quay.io/example/test-repo", - "result_reference": "quay.io/example-sign/sign-repo", - "sig_keyname": "testkey", - "signing_status": "failed", - "errors": ["error1", "error2"] + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "failed", + "errors": ["error1", "error2"] + } } event.message.body = json.dumps(test_failed_result) r_receiver.on_message(event) @@ -98,12 +112,19 @@ def test_radas_receiver(self): # test on_message: matched case with success status self.reset_receiver(r_receiver) test_success_result = { - "request_id": "test-request-id", - "file_reference": "quay.io/example/test-repo", - "result_reference": "quay.io/example-sign/sign-repo", - "sig_keyname": "testkey", - "signing_status": "success", - "errors": [] + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } } event.message.body = json.dumps(test_success_result) r_receiver.on_message(event) From 187f1ed462101a4bc7f4365fa196b235ac22ada0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 23 Jun 2025 15:46:31 +0800 Subject: [PATCH 36/64] control oras version under 0.2.31 to make it be compatible with python 3.9 --- pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 56148155..b667868d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", - "oras>=0.2.31", + "oras<=0.2.31", "python-qpid-proton>=0.39.0" ] diff --git a/requirements.txt b/requirements.txt index 75bb4b60..d5b5ec75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 -oras>=0.2.31 +oras<=0.2.31 python-qpid-proton>=0.39.0 \ No newline at end of file diff --git a/setup.py b/setup.py index ee5eac4b..3935d97b 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", - "oras>=0.2.31", + "oras<=0.2.31", "python-qpid-proton>=0.39.0" ], ) From 22009bf46668228865b9f5924411ed33aa71cee0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 25 Jun 2025 16:49:15 +0800 Subject: [PATCH 37/64] RADAS: fix a flag typo in upload for radas signing --- charon/cmd/cmd_upload.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index 2ad294f5..3a0e6990 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -137,13 +137,11 @@ ) @option("--dryrun", "-n", is_flag=True, default=False) @option( - "--sign_result_loc", + "--sign_result_file", "-l", - default="/tmp/sign", help=""" - The local save path for oras to pull the radas signature result. - Sign request will use this path to download the signature result, - Upload will use the file on this path to generate the corresponding .asc files + The path of the file which contains radas signature result. + Upload will use the file to generate the corresponding .asc files """, ) @command() From b7ec9781010a294254190e773b7726e9373aea5a Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 26 Jun 2025 17:57:48 +0800 Subject: [PATCH 38/64] Chore: RADAS: some logging adjustment --- charon/pkgs/radas_sign.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index bf63ff64..5f4e064e 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -24,6 +24,7 @@ from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import RadasConfig from charon.pkgs.oras_client import OrasClient +from charon.utils import files from proton import SSLDomain, Message, Event, Sender, Connection from proton.handlers import MessagingHandler from proton.reactor import Container @@ -327,20 +328,25 @@ async def generate_single_sign_file( signature_path = os.path.join(top_level, asc_filename) if not os.path.isfile(artifact_path): - logger.warning("Artifact missing, skip signature file generation") + logger.warning( + "Artifact %s missing, skip signature file generation.", + artifact_path) return try: - with open(signature_path, "w") as asc_file: - asc_file.write(signature) + files.overwrite_file(signature_path, signature) generated_signs.append(signature_path) - logger.info("Generated .asc file: %s", signature_path) + logger.debug("Generated .asc file: %s", signature_path) except Exception as e: failed_paths.append(signature_path) logger.error("Failed to write .asc file for %s: %s", artifact_path, e) result = data.get("results", []) - return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) + (_failed_metas, _generated_signs) = __do_path_cut_and(generate_single_sign_file, result) + logger.info( + "Signature generation done. There are %s signature files generated.", + len(_generated_signs)) + return (_failed_metas, _generated_signs) def __do_path_cut_and( From a9bcb5f05dbbb647425317552d74b6c3c8f087e3 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 26 Jun 2025 18:48:32 +0800 Subject: [PATCH 39/64] Fix: RADAS: fix a list index out of bounds issue --- charon/pkgs/radas_sign.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 5f4e064e..fcdf6e49 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -168,7 +168,8 @@ def _process_message(self, msg: Any) -> None: files = oras_client.pull( result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc ) - self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) + if files and len(files) > 0: + self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) else: self.log.error("The signing result received with failed status. Errors: %s", self.sign_result_errors) From cb220115de7674ed9d2a1321e42b274fb6e1883a Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 16 Jul 2025 15:08:47 +0800 Subject: [PATCH 40/64] Chore: some little adjustment * Increase the waiting time for CF invlidation check * Upgrade pytest to 8.4.1 --- charon/cache.py | 4 ++-- pyproject.toml | 2 +- tests/requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charon/cache.py b/charon/cache.py index 5b8d1227..3d216b60 100644 --- a/charon/cache.py +++ b/charon/cache.py @@ -86,8 +86,8 @@ def invalidate_paths( The default value is 3000 which is the maximum number in official doc: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits """ - INPRO_W_SECS = 5 - NEXT_W_SECS = 1 + INPRO_W_SECS = 10 + NEXT_W_SECS = 2 real_paths = [paths] # Split paths into batches by batch_size if batch_size: diff --git a/pyproject.toml b/pyproject.toml index b667868d..9061673b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ dev = [ test = [ "flexmock>=0.10.6", "responses>=0.9.0", - "pytest<=7.1.3", + "pytest<=8.4.1", "pytest-cov", "pytest-html", "requests-mock", diff --git a/tests/requirements.txt b/tests/requirements.txt index 408de626..477e546b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ flexmock>=0.10.6 responses>=0.9.0 -pytest<=7.1.3 +pytest<=8.4.1 pytest-cov pytest-html requests-mock From 8ae18613bbf73cbd8d2e045ecfd8fbcc48e2f366 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 25 Jun 2025 20:24:05 +0800 Subject: [PATCH 41/64] Chore: replace all file write with files.overwrite_file --- charon/pkgs/checksum_http.py | 16 ++++++++-------- charon/pkgs/indexing.py | 5 ++--- charon/utils/files.py | 3 +-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/charon/pkgs/checksum_http.py b/charon/pkgs/checksum_http.py index e57dab34..e30a373e 100644 --- a/charon/pkgs/checksum_http.py +++ b/charon/pkgs/checksum_http.py @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from charon.utils.files import digest, HashType +from charon.utils.files import digest, HashType, overwrite_file from charon.storage import S3Client from typing import Tuple, List, Dict, Optional from html.parser import HTMLParser @@ -169,9 +169,10 @@ def _check_and_remove_file(file_name: str): def _write_one_col_file(items: List[str], file_name: str): if items and len(items) > 0: _check_and_remove_file(file_name) - with open(file_name, "w") as f: - for i in items: - f.write(i + "\n") + content = "" + for i in items: + content = content + i + "\n" + overwrite_file(file_name, content) logger.info("The report file %s is generated.", file_name) _write_one_col_file(content[0], os.path.join(work_dir, "mismatched_files.csv")) @@ -180,10 +181,9 @@ def _write_one_col_file(items: List[str], file_name: str): if content[2] and len(content[2]) > 0: error_file = os.path.join(work_dir, "error_files.csv") _check_and_remove_file(error_file) - with open(error_file, "w") as f: - f.write("path,error\n") - for d in content[2]: - f.write("{path},{error}\n".format(path=d["path"], error=d["error"])) + f_content_lines: List[str] = [] + f_content = "path,error\n" + "\n".join(f_content_lines) + overwrite_file(error_file, f_content) logger.info("The report file %s is generated.", error_file) diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index 4710cdab..d0e70638 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -19,7 +19,7 @@ # from charon.pkgs.pkg_utils import invalidate_cf_paths from charon.constants import (INDEX_HTML_TEMPLATE, NPM_INDEX_HTML_TEMPLATE, PACKAGE_TYPE_MAVEN, PACKAGE_TYPE_NPM, PROD_INFO_SUFFIX) -from charon.utils.files import digest_content +from charon.utils.files import digest_content, overwrite_file from jinja2 import Template import os import logging @@ -155,8 +155,7 @@ def __to_html(package_type: str, contents: List[str], folder: str, top_level: st if folder == "/": html_path = os.path.join(top_level, "index.html") os.makedirs(os.path.dirname(html_path), exist_ok=True) - with open(html_path, 'w', encoding='utf-8') as html: - html.write(html_content) + overwrite_file(html_path, html_content) return html_path diff --git a/charon/utils/files.py b/charon/utils/files.py index d811200b..ccad3e23 100644 --- a/charon/utils/files.py +++ b/charon/utils/files.py @@ -125,6 +125,5 @@ def write_manifest(paths: List[str], root: str, product_key: str) -> Tuple[str, if not os.path.isfile(manifest_path): with open(manifest_path, mode="a", encoding="utf-8"): pass - with open(manifest_path, mode="w", encoding="utf-8") as f: - f.write('\n'.join(artifacts)) + overwrite_file(manifest_path, '\n'.join(artifacts)) return manifest_name, manifest_path From ecb572491b2524c35059b29ffae064523e035e2a Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 9 Jul 2025 10:12:54 +0800 Subject: [PATCH 42/64] Feat: support recursive indexing for index function --- charon/cmd/cmd_index.py | 18 +++++++++++++++++- charon/pkgs/indexing.py | 24 +++++++++++++++++++----- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/charon/cmd/cmd_index.py b/charon/cmd/cmd_index.py index e5dd11a5..7d4c07a6 100644 --- a/charon/cmd/cmd_index.py +++ b/charon/cmd/cmd_index.py @@ -42,6 +42,13 @@ """, required=True ) +@option( + "--recursive", + "-r", + help="If do indexing recursively under $path", + is_flag=True, + default=False +) @option( "--config", "-c", @@ -69,6 +76,7 @@ def index( path: str, target: str, + recursive: bool = False, config: str = None, debug: bool = False, quiet: bool = False, @@ -120,7 +128,15 @@ def index( if not aws_bucket: logger.error("No bucket specified for target %s!", target) else: - re_index(b, path, package_type, aws_profile, dryrun) + args = { + "target": b, + "path": path, + "package_type": package_type, + "aws_profile": aws_profile, + "recursive": recursive, + "dry_run": dryrun + } + re_index(**args) # type: ignore except Exception: print(traceback.format_exc()) diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index d0e70638..6794a478 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -266,7 +266,7 @@ def re_index( path: str, package_type: str, aws_profile: str = None, - # cf_enable: bool = False, + recursive: bool = False, dry_run: bool = False ): """Refresh the index.html for the specified folder in the bucket. @@ -306,6 +306,7 @@ def re_index( logger.debug("The re-indexed page content: %s", index_content) if not dry_run: index_path = os.path.join(path, "index.html") + logger.info("Start re-indexing %s in bucket %s", index_path, bucket_name) if path == "/": index_path = "index.html" s3_client.simple_delete_file(index_path, (bucket_name, real_prefix)) @@ -313,10 +314,23 @@ def re_index( index_path, index_content, (bucket_name, real_prefix), "text/html", digest_content(index_content) ) - # We will not invalidate index.html per cost consideration - # if cf_enable: - # cf_client = CFClient(aws_profile=aws_profile) - # invalidate_cf_paths(cf_client, bucket, [index_path]) + logger.info("%s re-indexing finished", index_path) + if recursive: + for c in contents: + if c.endswith("/"): + sub_path = c.removeprefix(real_prefix).strip() + if sub_path.startswith("/"): + sub_path = sub_path.removeprefix("/") + logger.debug("subpath: %s", sub_path) + args = { + "target": target, + "path": sub_path, + "package_type": package_type, + "aws_profile": aws_profile, + "recursive": recursive, + "dry_run": dry_run + } + re_index(**args) # type: ignore else: logger.warning( "The path %s does not contain any contents in bucket %s. " From cc52352e16335a6c1349610b9c0a18ceb00212b5 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 19 Sep 2025 10:09:22 +0800 Subject: [PATCH 43/64] Fix a issue of pyproject.toml file --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8e211380..7fa24200 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license-files = ["LICENSE"] +license = {file="LICENSE"} requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", From 308f53aadaa95f92cbf2cea3dae6066e8a62d53d Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 19 Sep 2025 11:28:06 +0800 Subject: [PATCH 44/64] Some fix for setup tools * Add MANIFEST.in for sdist generation * Fix some warning for sdist build --- MANIFEST.in | 8 ++++++++ pyproject.toml | 7 +++---- setup.py | 29 +++++++++++++---------------- 3 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..6a2a3c77 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ + include LICENSE + include README.md + include pyproject.toml + include setup.py + recursive-include charon *.py *.json + recursive-include tests *.py *.txt *.tgz *.zip *.json *.sha1 + exclude .github .gitignore + diff --git a/pyproject.toml b/pyproject.toml index 7fa24200..2d2764e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,11 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license = {file="LICENSE"} +license = "Apache-2.0" requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", "Programming Language :: Python :: 3 :: Only", @@ -61,8 +60,8 @@ test = [ [project.scripts] charon = "charon.cmd:cli" -[tool.setuptools] -packages = ["charon"] +[tool.setuptools.packages.find] +include = ["charon*"] [tool.setuptools_scm] fallback_version = "1.3.4+dev.fallback" diff --git a/setup.py b/setup.py index 934ae861..5c37fc84 100755 --- a/setup.py +++ b/setup.py @@ -32,31 +32,28 @@ classifiers=[ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", ], keywords="charon mrrc maven npm build java", author="RedHat EXD SPMM", - license="APLv2", packages=find_packages(exclude=["ez_setup", "examples", "tests"]), package_data={'charon': ['schemas/*.json']}, - test_suite="tests", entry_points={ "console_scripts": ["charon = charon.cmd:cli"], }, - install_requires=[ - "Jinja2>=3.1.3", - "boto3>=1.18.35", - "botocore>=1.21.35", - "click>=8.1.3", - "requests>=2.25.0", - "PyYAML>=5.4.1", - "defusedxml>=0.7.1", - "subresource-integrity>=0.2", - "jsonschema>=4.9.1", - "urllib3>=1.25.10", - "semantic-version>=2.10.0" - ], + # install_requires=[ + # "Jinja2>=3.1.3", + # "boto3>=1.18.35", + # "botocore>=1.21.35", + # "click>=8.1.3", + # "requests>=2.25.0", + # "PyYAML>=5.4.1", + # "defusedxml>=0.7.1", + # "subresource-integrity>=0.2", + # "jsonschema>=4.9.1", + # "urllib3>=1.25.10", + # "semantic-version>=2.10.0" + # ], ) From 195291299de7d68104e06cc64dad45ff052e7432 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 19 Sep 2025 10:09:22 +0800 Subject: [PATCH 45/64] Fix a issue of pyproject.toml file --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9061673b..68fc5461 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license-files = ["LICENSE"] +license = {file="LICENSE"} requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", From 8ad7932532ce95497cfb808ab9d484351b0fbb99 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 19 Sep 2025 11:28:06 +0800 Subject: [PATCH 46/64] Some fix for setup tools * Add MANIFEST.in for sdist generation * Fix some warning for sdist build --- MANIFEST.in | 8 ++++++++ pyproject.toml | 7 +++---- setup.py | 31 +++++++++++++------------------ 3 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..6a2a3c77 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ + include LICENSE + include README.md + include pyproject.toml + include setup.py + recursive-include charon *.py *.json + recursive-include tests *.py *.txt *.tgz *.zip *.json *.sha1 + exclude .github .gitignore + diff --git a/pyproject.toml b/pyproject.toml index 68fc5461..310bb1f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,11 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license = {file="LICENSE"} +license = "Apache-2.0" requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", "Programming Language :: Python :: 3 :: Only", @@ -63,8 +62,8 @@ test = [ [project.scripts] charon = "charon.cmd:cli" -[tool.setuptools] -packages = ["charon"] +[tool.setuptools.packages.find] +include = ["charon*"] [tool.setuptools_scm] fallback_version = "1.3.4+dev.fallback" diff --git a/setup.py b/setup.py index 3935d97b..2de9c2e6 100755 --- a/setup.py +++ b/setup.py @@ -32,33 +32,28 @@ classifiers=[ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", ], keywords="charon mrrc maven npm build java", author="RedHat EXD SPMM", - license="APLv2", packages=find_packages(exclude=["ez_setup", "examples", "tests"]), package_data={'charon': ['schemas/*.json']}, - test_suite="tests", entry_points={ "console_scripts": ["charon = charon.cmd:cli"], }, - install_requires=[ - "Jinja2>=3.1.3", - "boto3>=1.18.35", - "botocore>=1.21.35", - "click>=8.1.3", - "requests>=2.25.0", - "PyYAML>=5.4.1", - "defusedxml>=0.7.1", - "subresource-integrity>=0.2", - "jsonschema>=4.9.1", - "urllib3>=1.25.10", - "semantic-version>=2.10.0", - "oras<=0.2.31", - "python-qpid-proton>=0.39.0" - ], + # install_requires=[ + # "Jinja2>=3.1.3", + # "boto3>=1.18.35", + # "botocore>=1.21.35", + # "click>=8.1.3", + # "requests>=2.25.0", + # "PyYAML>=5.4.1", + # "defusedxml>=0.7.1", + # "subresource-integrity>=0.2", + # "jsonschema>=4.9.1", + # "urllib3>=1.25.10", + # "semantic-version>=2.10.0" + # ], ) From 00e7e19d2a2de0dcacee2a68e93cf3a96f89d286 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 19 Sep 2025 12:50:28 +0800 Subject: [PATCH 47/64] Fix pyproject.toml license issue --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d2764e1..cd439657 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license = "Apache-2.0" +license = {text="Apache-2.0"} requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", From d673970752eea1fe84ad1ba762fa2896f65e3c63 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 19 Sep 2025 12:50:28 +0800 Subject: [PATCH 48/64] Fix pyproject.toml license issue --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 310bb1f9..4f87614b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license = "Apache-2.0" +license = {text="Apache-2.0"} requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", From 29594e57b7d9b31df6757af896b5db8eaeb6415b Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 13 Oct 2025 15:50:03 +0800 Subject: [PATCH 49/64] Feat: Accept multiple maven zips with non-RADAS signing way --- charon/cmd/cmd_upload.py | 65 ++++++++------- charon/cmd/internal.py | 8 ++ charon/pkgs/maven.py | 140 ++++++++++++++++++++++++++++++++- charon/utils/archive.py | 13 +++ tests/test_archive.py | 33 +++++++- tests/test_extract_tarballs.py | 31 ++++++++ tests/test_maven_upload.py | 62 +++++++++++++++ 7 files changed, 320 insertions(+), 32 deletions(-) create mode 100644 tests/test_extract_tarballs.py diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a867df01..dcc24a88 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -16,12 +16,12 @@ from typing import List from charon.config import get_config -from charon.utils.archive import detect_npm_archive, NpmArchiveType +from charon.utils.archive import detect_npm_archives, NpmArchiveType from charon.pkgs.maven import handle_maven_uploading from charon.pkgs.npm import handle_npm_uploading from charon.cmd.internal import ( _decide_mode, _validate_prod_key, - _get_local_repo, _get_targets, + _get_local_repos, _get_targets, _get_ignore_patterns, _safe_delete ) from click import command, option, argument @@ -35,8 +35,10 @@ @argument( - "repo", + "repos", type=str, + nargs='+', # This allows multiple arguments for zip urls + required=True ) @option( "--product", @@ -138,7 +140,7 @@ @option("--dryrun", "-n", is_flag=True, default=False) @command() def upload( - repo: str, + repos: List[str], product: str, version: str, targets: List[str], @@ -152,9 +154,9 @@ def upload( quiet=False, dryrun=False ): - """Upload all files from a released product REPO to Ronda - Service. The REPO points to a product released tarball which - is hosted in a remote url or a local path. + """Upload all files from released product REPOs to Ronda + Service. The REPOs point to a product released tarballs which + are hosted in remote urls or local paths. """ tmp_dir = work_dir try: @@ -173,8 +175,8 @@ def upload( logger.error("No AWS profile specified!") sys.exit(1) - archive_path = _get_local_repo(repo) - npm_archive_type = detect_npm_archive(archive_path) + archive_paths = _get_local_repos(repos) + archive_types = detect_npm_archives(archive_paths) product_key = f"{product}-{version}" manifest_bucket_name = conf.get_manifest_bucket() targets_ = _get_targets(targets, conf) @@ -185,23 +187,10 @@ def upload( " are set correctly.", targets_ ) sys.exit(1) - if npm_archive_type != NpmArchiveType.NOT_NPM: - logger.info("This is a npm archive") - tmp_dir, succeeded = handle_npm_uploading( - archive_path, - product_key, - targets=targets_, - aws_profile=aws_profile, - dir_=work_dir, - gen_sign=contain_signature, - cf_enable=conf.is_aws_cf_enable(), - key=sign_key, - dry_run=dryrun, - manifest_bucket_name=manifest_bucket_name - ) - if not succeeded: - sys.exit(1) - else: + + maven_count = archive_types.count(NpmArchiveType.NOT_NPM) + npm_count = len(archive_types) - maven_count + if maven_count == len(archive_types): ignore_patterns_list = None if ignore_patterns: ignore_patterns_list = ignore_patterns @@ -209,7 +198,7 @@ def upload( ignore_patterns_list = _get_ignore_patterns(conf) logger.info("This is a maven archive") tmp_dir, succeeded = handle_maven_uploading( - archive_path, + archive_paths, product_key, ignore_patterns_list, root=root_path, @@ -225,6 +214,28 @@ def upload( ) if not succeeded: sys.exit(1) + elif npm_count == len(archive_types) and len(archive_types) == 1: + logger.info("This is a npm archive") + tmp_dir, succeeded = handle_npm_uploading( + archive_paths[0], + product_key, + targets=targets_, + aws_profile=aws_profile, + dir_=work_dir, + gen_sign=contain_signature, + cf_enable=conf.is_aws_cf_enable(), + key=sign_key, + dry_run=dryrun, + manifest_bucket_name=manifest_bucket_name + ) + if not succeeded: + sys.exit(1) + elif npm_count == len(archive_types) and len(archive_types) > 1: + logger.error("Doesn't support multiple upload for npm") + sys.exit(1) + else: + logger.error("Upload types are not consistent") + sys.exit(1) except Exception: print(traceback.format_exc()) sys.exit(2) # distinguish between exception and bad config or bad state diff --git a/charon/cmd/internal.py b/charon/cmd/internal.py index e7e7d14a..89d4ea1b 100644 --- a/charon/cmd/internal.py +++ b/charon/cmd/internal.py @@ -75,6 +75,14 @@ def _get_local_repo(url: str) -> str: return archive_path +def _get_local_repos(urls: list) -> list: + archive_paths = [] + for url in urls: + archive_path = _get_local_repo(url) + archive_paths.append(archive_path) + return archive_paths + + def _validate_prod_key(product: str, version: str) -> bool: if not product or product.strip() == "": logger.error("Error: product can not be empty!") diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9f50f35b..b183c474 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -32,11 +32,12 @@ META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME, PACKAGE_TYPE_MAVEN) -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Union from jinja2 import Template from datetime import datetime from zipfile import ZipFile, BadZipFile from tempfile import mkdtemp +from shutil import rmtree, copy2 from defusedxml import ElementTree import os @@ -261,7 +262,7 @@ def __gen_digest_file(hash_file_path, meta_file_path: str, hashtype: HashType) - def handle_maven_uploading( - repo: str, + repos: Union[str, List[str]], prod_key: str, ignore_patterns=None, root="maven-repository", @@ -294,8 +295,10 @@ def handle_maven_uploading( """ if targets is None: targets = [] - # 1. extract tarball - tmp_root = _extract_tarball(repo, prod_key, dir__=dir_) + if isinstance(repos, str): + repos = [repos] + # 1. extract tarballs + tmp_root = _extract_tarballs(repos, root, prod_key, dir__=dir_) # 2. scan for paths and filter out the ignored paths, # and also collect poms for later metadata generation @@ -673,6 +676,135 @@ def _extract_tarball(repo: str, prefix="", dir__=None) -> str: sys.exit(1) +def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str: + """ Extract multiple zip archives to a temporary directory. + * repos are the list of repo paths to extract + * root is a prefix in the tarball to identify which path is + the beginning of the maven GAV path + * prefix is the prefix for temporary directory name + * dir__ is the directory where temporary directories will be created. + + Returns the path to the merged temporary directory containing all extracted files + """ + # Create final merge directory + final_tmp_root = mkdtemp(prefix=f"charon-{prefix}-final-", dir=dir__) + + total_copied = 0 + total_overwritten = 0 + total_processed = 0 + + # Collect all extracted directories first + extracted_dirs = [] + + for repo in repos: + if os.path.exists(repo): + try: + logger.info("Extracting tarball %s", repo) + repo_zip = ZipFile(repo) + tmp_root = mkdtemp(prefix=f"charon-{prefix}-", dir=dir__) + extract_zip_all(repo_zip, tmp_root) + extracted_dirs.append(tmp_root) + + except BadZipFile as e: + logger.error("Tarball extraction error: %s", e) + sys.exit(1) + else: + logger.error("Error: archive %s does not exist", repo) + sys.exit(1) + + # Merge all extracted directories + if extracted_dirs: + # Get top-level directory names for merged from all repos + top_level_merged_name_dirs = [] + for extracted_dir in extracted_dirs: + for item in os.listdir(extracted_dir): + item_path = os.path.join(extracted_dir, item) + # Check the root maven-repository subdirectory existence + maven_repo_path = os.path.join(item_path, root) + if os.path.isdir(item_path) and os.path.exists(maven_repo_path): + top_level_merged_name_dirs.append(item) + break + + # Create merged directory name + merged_dir_name = ( + "_".join(top_level_merged_name_dirs) if top_level_merged_name_dirs else "merged" + ) + merged_dest_dir = os.path.join(final_tmp_root, merged_dir_name) + + # Merge content from all extracted directories + for extracted_dir in extracted_dirs: + copied, overwritten, processed = _merge_directories_with_rename( + extracted_dir, merged_dest_dir, root + ) + total_copied += copied + total_overwritten += overwritten + total_processed += processed + + # Clean up temporary extraction directory + rmtree(extracted_dir) + + logger.info( + "All zips merged! Total copied: %s, Total overwritten: %s, Total processed: %s", + total_copied, + total_overwritten, + total_processed, + ) + return final_tmp_root + + +def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): + """ Recursively copy files from src_dir to dest_dir, overwriting existing files. + * src_dir is the source directory to copy from + * dest_dir is the destination directory to copy to. + + Returns Tuple of (copied_count, overwritten_count, processed_count) + """ + copied_count = 0 + overwritten_count = 0 + processed_count = 0 + + # Find the actual content directory + content_root = src_dir + for item in os.listdir(src_dir): + item_path = os.path.join(src_dir, item) + # Check the root maven-repository subdirectory existence + maven_repo_path = os.path.join(item_path, root) + if os.path.isdir(item_path) and os.path.exists(maven_repo_path): + content_root = item_path + break + + # pylint: disable=unused-variable + for root_dir, dirs, files in os.walk(content_root): + # Calculate relative path from content root + rel_path = os.path.relpath(root_dir, content_root) + dest_root = os.path.join(dest_dir, rel_path) if rel_path != '.' else dest_dir + + # Create destination directory if it doesn't exist + os.makedirs(dest_root, exist_ok=True) + + # Copy all files, overwriting existing ones + for file in files: + src_file = os.path.join(root_dir, file) + dest_file = os.path.join(dest_root, file) + if os.path.exists(dest_file): + overwritten_count += 1 + logger.debug("Overwritten: %s -> %s", src_file, dest_file) + else: + copied_count += 1 + logger.debug("Copied: %s -> %s", src_file, dest_file) + + processed_count += 1 + copy2(src_file, dest_file) + + logger.info( + "One zip merged! Files copied: %s, Files overwritten: %s, Total files processed: %s", + copied_count, + overwritten_count, + processed_count, + ) + return copied_count, overwritten_count, processed_count + + def _scan_paths(files_root: str, ignore_patterns: List[str], root: str) -> Tuple[str, List[str], List[str], List[str]]: # 2. scan for paths and filter out the ignored paths, diff --git a/charon/utils/archive.py b/charon/utils/archive.py index 4a1f256c..058fa17e 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -182,6 +182,19 @@ def detect_npm_archive(repo): return NpmArchiveType.NOT_NPM +def detect_npm_archives(repos): + """Detects, if the archives need to have npm workflow. + :parameter repos list of repository directories + :return list of NpmArchiveType values + """ + results = [] + for repo in repos: + result = detect_npm_archive(repo) + results.append(result) + + return results + + def download_archive(url: str, base_dir=None) -> str: dir_ = base_dir if not dir_ or not os.path.isdir(dir_): diff --git a/tests/test_archive.py b/tests/test_archive.py index 0e2ac09a..22cf48fd 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -1,5 +1,5 @@ from tests.base import BaseTest -from charon.utils.archive import NpmArchiveType, detect_npm_archive +from charon.utils.archive import NpmArchiveType, detect_npm_archive, detect_npm_archives import os from tests.constants import INPUTS @@ -12,5 +12,36 @@ def test_detect_package(self): npm_tarball = os.path.join(INPUTS, "code-frame-7.14.5.tgz") self.assertEqual(NpmArchiveType.TAR_FILE, detect_npm_archive(npm_tarball)) + def test_detect_packages(self): + mvn_tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip") + ] + archive_types = detect_npm_archives(mvn_tarballs) + self.assertEqual(2, archive_types.count(NpmArchiveType.NOT_NPM)) + + npm_tarball = [ + os.path.join(INPUTS, "code-frame-7.14.5.tgz") + ] + archive_types = detect_npm_archives(npm_tarball) + self.assertEqual(1, archive_types.count(NpmArchiveType.TAR_FILE)) + + npm_tarballs = [ + os.path.join(INPUTS, "code-frame-7.14.5.tgz"), + os.path.join(INPUTS, "code-frame-7.15.8.tgz") + ] + archive_types = detect_npm_archives(npm_tarballs) + self.assertEqual(2, archive_types.count(NpmArchiveType.TAR_FILE)) + + tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip"), + os.path.join(INPUTS, "code-frame-7.14.5.tgz"), + os.path.join(INPUTS, "code-frame-7.15.8.tgz") + ] + archive_types = detect_npm_archives(tarballs) + self.assertEqual(2, archive_types.count(NpmArchiveType.NOT_NPM)) + self.assertEqual(2, archive_types.count(NpmArchiveType.TAR_FILE)) + def test_download_archive(self): pass diff --git a/tests/test_extract_tarballs.py b/tests/test_extract_tarballs.py new file mode 100644 index 00000000..22190bfc --- /dev/null +++ b/tests/test_extract_tarballs.py @@ -0,0 +1,31 @@ +from tests.base import BaseTest +from charon.pkgs.maven import _extract_tarballs +import os + +from tests.constants import INPUTS + + +class ArchiveTest(BaseTest): + def test_extract_tarballs(self): + mvn_tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip"), + ] + final_merged_path = _extract_tarballs(mvn_tarballs, "maven-repository") + expected_dir = os.path.join( + final_merged_path, "commons-client-4.5.6_commons-client-4.5.9", "maven-repository" + ) + self.assertTrue(os.path.exists(expected_dir)) + + expected_files = [ + "org/apache/httpcomponents/httpclient/4.5.9/httpclient-4.5.9.jar", + "org/apache/httpcomponents/httpclient/4.5.9/httpclient-4.5.9.pom", + "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.jar", + "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom", + ] + for expected_file in expected_files: + file_path = os.path.join(expected_dir, expected_file) + self.assertTrue(os.path.exists(file_path)) + + def test_download_archive(self): + pass diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index 629a9e3f..6f40a8ca 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -110,6 +110,68 @@ def test_overlap_upload(self): self.assertIn("httpclient", cat_content) self.assertIn("org.apache.httpcomponents", cat_content) + def test_multi_zips_upload(self): + mvn_tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip") + ] + product_45 = "commons-client-4.5" + + handle_maven_uploading( + mvn_tarballs, product_45, + targets=[('', TEST_BUCKET, '', '')], + dir_=self.tempdir, do_index=False + ) + + objs = list(self.test_bucket.objects.all()) + actual_files = [obj.key for obj in objs] + # need to double mvn num because of .prodinfo files + self.assertEqual( + COMMONS_CLIENT_MVN_NUM * 2 + COMMONS_CLIENT_META_NUM, + len(actual_files) + ) + + filesets = [ + COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES, + COMMONS_CLIENT_459_FILES, + ARCHETYPE_CATALOG_FILES + ] + for fileset in filesets: + for f in fileset: + self.assertIn(f, actual_files) + + product_mix = [product_45] + for f in COMMONS_LOGGING_FILES: + self.assertIn(f, actual_files) + self.check_product(f, product_mix) + for f in COMMONS_LOGGING_METAS: + self.assertIn(f, actual_files) + + meta_obj_client = self.test_bucket.Object(COMMONS_CLIENT_METAS[0]) + meta_content_client = str(meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn( + "org.apache.httpcomponents", meta_content_client + ) + self.assertIn("httpclient", meta_content_client) + self.assertIn("4.5.9", meta_content_client) + self.assertIn("4.5.9", meta_content_client) + self.assertIn("4.5.6", meta_content_client) + self.assertIn("4.5.9", meta_content_client) + + meta_obj_logging = self.test_bucket.Object(COMMONS_LOGGING_METAS[0]) + meta_content_logging = str(meta_obj_logging.get()["Body"].read(), "utf-8") + self.assertIn("commons-logging", meta_content_logging) + self.assertIn("commons-logging", meta_content_logging) + self.assertIn("1.2", meta_content_logging) + self.assertIn("1.2", meta_content_logging) + self.assertIn("1.2", meta_content_logging) + + catalog = self.test_bucket.Object(ARCHETYPE_CATALOG) + cat_content = str(catalog.get()["Body"].read(), "utf-8") + self.assertIn("4.5.9", cat_content) + self.assertIn("httpclient", cat_content) + self.assertIn("org.apache.httpcomponents", cat_content) + def test_ignore_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" From 6d4dc56a59101ca38a3bb52de57badb6fc586741 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 14 Oct 2025 14:15:14 +0800 Subject: [PATCH 50/64] Fix TypeError for argument multi nargs value defination --- charon/cmd/cmd_upload.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index dcc24a88..4cd0d8dd 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -37,8 +37,7 @@ @argument( "repos", type=str, - nargs='+', # This allows multiple arguments for zip urls - required=True + nargs=-1 # This allows multiple arguments for zip urls ) @option( "--product", From f3a2c55d9233895f39d330aae3120fde3a7e17cc Mon Sep 17 00:00:00 2001 From: yma Date: Fri, 17 Oct 2025 11:54:43 +0800 Subject: [PATCH 51/64] Fix note, err log, repos param list type, merged dir name length, typo issues --- charon/cmd/cmd_upload.py | 1 + charon/pkgs/maven.py | 24 +++++------------------- tests/test_cf_maven_ops.py | 4 ++-- tests/test_cf_reindex.py | 2 +- tests/test_extract_tarballs.py | 5 +---- tests/test_manifest_del.py | 2 +- tests/test_manifest_upload.py | 2 +- tests/test_maven_del.py | 4 ++-- tests/test_maven_del_multi_tgts.py | 4 ++-- tests/test_maven_index.py | 14 +++++++------- tests/test_maven_index_multi_tgts.py | 12 ++++++------ tests/test_maven_sign.py | 6 +++--- tests/test_maven_upload.py | 8 ++++---- tests/test_maven_upload_multi_tgts.py | 8 ++++---- tests/test_pkgs_dryrun.py | 6 +++--- 15 files changed, 43 insertions(+), 59 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index 4cd0d8dd..f1e4df3f 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -156,6 +156,7 @@ def upload( """Upload all files from released product REPOs to Ronda Service. The REPOs point to a product released tarballs which are hosted in remote urls or local paths. + Notes: It does not support multiple repos for NPM archives """ tmp_dir = work_dir try: diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index b183c474..90724050 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -32,7 +32,7 @@ META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME, PACKAGE_TYPE_MAVEN) -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Tuple from jinja2 import Template from datetime import datetime from zipfile import ZipFile, BadZipFile @@ -262,7 +262,7 @@ def __gen_digest_file(hash_file_path, meta_file_path: str, hashtype: HashType) - def handle_maven_uploading( - repos: Union[str, List[str]], + repos: List[str], prod_key: str, ignore_patterns=None, root="maven-repository", @@ -295,8 +295,7 @@ def handle_maven_uploading( """ if targets is None: targets = [] - if isinstance(repos, str): - repos = [repos] + # 1. extract tarballs tmp_root = _extract_tarballs(repos, root, prod_key, dir__=dir_) @@ -706,7 +705,7 @@ def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str extracted_dirs.append(tmp_root) except BadZipFile as e: - logger.error("Tarball extraction error: %s", e) + logger.error("Tarball extraction error for repo %s: %s", repo, e) sys.exit(1) else: logger.error("Error: archive %s does not exist", repo) @@ -714,21 +713,8 @@ def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str # Merge all extracted directories if extracted_dirs: - # Get top-level directory names for merged from all repos - top_level_merged_name_dirs = [] - for extracted_dir in extracted_dirs: - for item in os.listdir(extracted_dir): - item_path = os.path.join(extracted_dir, item) - # Check the root maven-repository subdirectory existence - maven_repo_path = os.path.join(item_path, root) - if os.path.isdir(item_path) and os.path.exists(maven_repo_path): - top_level_merged_name_dirs.append(item) - break - # Create merged directory name - merged_dir_name = ( - "_".join(top_level_merged_name_dirs) if top_level_merged_name_dirs else "merged" - ) + merged_dir_name = "merged_repositories" merged_dest_dir = os.path.join(final_tmp_root, merged_dir_name) # Merge content from all extracted directories diff --git a/tests/test_cf_maven_ops.py b/tests/test_cf_maven_ops.py index b8cb03c1..ca5ac361 100644 --- a/tests/test_cf_maven_ops.py +++ b/tests/test_cf_maven_ops.py @@ -31,7 +31,7 @@ def test_cf_after_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True, @@ -52,7 +52,7 @@ def test_cf_after_del(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True diff --git a/tests/test_cf_reindex.py b/tests/test_cf_reindex.py index 944a86f2..941793fd 100644 --- a/tests/test_cf_reindex.py +++ b/tests/test_cf_reindex.py @@ -40,7 +40,7 @@ def test_cf_maven_after_reindex(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir ) diff --git a/tests/test_extract_tarballs.py b/tests/test_extract_tarballs.py index 22190bfc..53a96f63 100644 --- a/tests/test_extract_tarballs.py +++ b/tests/test_extract_tarballs.py @@ -13,7 +13,7 @@ def test_extract_tarballs(self): ] final_merged_path = _extract_tarballs(mvn_tarballs, "maven-repository") expected_dir = os.path.join( - final_merged_path, "commons-client-4.5.6_commons-client-4.5.9", "maven-repository" + final_merged_path, "merged_repositories", "maven-repository" ) self.assertTrue(os.path.exists(expected_dir)) @@ -26,6 +26,3 @@ def test_extract_tarballs(self): for expected_file in expected_files: file_path = os.path.join(expected_dir, expected_file) self.assertTrue(os.path.exists(file_path)) - - def test_download_archive(self): - pass diff --git a/tests/test_manifest_del.py b/tests/test_manifest_del.py index 7a81be3c..c47c7602 100644 --- a/tests/test_manifest_del.py +++ b/tests/test_manifest_del.py @@ -77,7 +77,7 @@ def __prepare_maven_content(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, diff --git a/tests/test_manifest_upload.py b/tests/test_manifest_upload.py index c7e801b2..520f0679 100644 --- a/tests/test_manifest_upload.py +++ b/tests/test_manifest_upload.py @@ -36,7 +36,7 @@ def test_maven_manifest_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index 5b565adc..86425724 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -190,7 +190,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False @@ -199,7 +199,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False diff --git a/tests/test_maven_del_multi_tgts.py b/tests/test_maven_del_multi_tgts.py index 26fa11cc..2a7d042f 100644 --- a/tests/test_maven_del_multi_tgts.py +++ b/tests/test_maven_del_multi_tgts.py @@ -259,7 +259,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" targets_ = [('', TEST_BUCKET, prefix, ''), ('', TEST_BUCKET_2, prefix, '')] handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir, do_index=False @@ -268,7 +268,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir, do_index=False diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index a5cd1ed2..33533337 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -37,7 +37,7 @@ def test_uploading_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -79,7 +79,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -87,7 +87,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -130,7 +130,7 @@ def test_re_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -221,7 +221,7 @@ def __test_upload_index_with_prefix(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -403,7 +403,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -411,7 +411,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) diff --git a/tests/test_maven_index_multi_tgts.py b/tests/test_maven_index_multi_tgts.py index cc9d0718..44f921bf 100644 --- a/tests/test_maven_index_multi_tgts.py +++ b/tests/test_maven_index_multi_tgts.py @@ -46,7 +46,7 @@ def test_uploading_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=targets_, dir_=self.tempdir ) @@ -106,7 +106,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir ) @@ -114,7 +114,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir ) @@ -194,7 +194,7 @@ def __test_upload_index_with_prefix(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=targets_, dir_=self.tempdir ) @@ -417,7 +417,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir ) @@ -425,7 +425,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir ) diff --git a/tests/test_maven_sign.py b/tests/test_maven_sign.py index f60ee54d..834326bf 100644 --- a/tests/test_maven_sign.py +++ b/tests/test_maven_sign.py @@ -32,7 +32,7 @@ def test_uploading_sign(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, @@ -63,7 +63,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, @@ -73,7 +73,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index 6f40a8ca..fefa74ea 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -47,7 +47,7 @@ def test_overlap_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -55,7 +55,7 @@ def test_overlap_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -176,7 +176,7 @@ def test_ignore_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, [".*.sha1"], + [test_zip], product_456, [".*.sha1"], targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -205,7 +205,7 @@ def __test_prefix_upload(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False diff --git a/tests/test_maven_upload_multi_tgts.py b/tests/test_maven_upload_multi_tgts.py index 35aa49d4..f6eb289e 100644 --- a/tests/test_maven_upload_multi_tgts.py +++ b/tests/test_maven_upload_multi_tgts.py @@ -68,7 +68,7 @@ def test_overlap_upload(self): ('', TEST_BUCKET, '', ''), ('', TEST_BUCKET_2, '', '') ] handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir, do_index=False ) @@ -76,7 +76,7 @@ def test_overlap_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir, do_index=False ) @@ -186,7 +186,7 @@ def test_ignore_upload(self): ('', TEST_BUCKET, '', ''), ('', TEST_BUCKET_2, '', '') ] handle_maven_uploading( - test_zip, product_456, [".*.sha1"], + [test_zip], product_456, [".*.sha1"], targets=targets_, dir_=self.tempdir, do_index=False ) @@ -221,7 +221,7 @@ def __test_prefix_upload(self, targets: List[Tuple[str, str, str, str]]): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=targets, dir_=self.tempdir, do_index=False diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index 46061734..c49ad14d 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -30,7 +30,7 @@ def test_maven_upload_dry_run(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, dry_run=True @@ -90,7 +90,7 @@ def __prepare_maven_content(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -98,7 +98,7 @@ def __prepare_maven_content(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) From 45d3210bfa84849701fd3edb7e977f78c90e8e8d Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 22 Oct 2025 10:27:35 +0800 Subject: [PATCH 52/64] Fix files duplicated logic for the merge overlapping case --- charon/pkgs/maven.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 90724050..6ac406c5 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -689,7 +689,7 @@ def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str final_tmp_root = mkdtemp(prefix=f"charon-{prefix}-final-", dir=dir__) total_copied = 0 - total_overwritten = 0 + total_duplicated = 0 total_processed = 0 # Collect all extracted directories first @@ -719,20 +719,20 @@ def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str # Merge content from all extracted directories for extracted_dir in extracted_dirs: - copied, overwritten, processed = _merge_directories_with_rename( + copied, duplicated, processed = _merge_directories_with_rename( extracted_dir, merged_dest_dir, root ) total_copied += copied - total_overwritten += overwritten + total_duplicated += duplicated total_processed += processed # Clean up temporary extraction directory rmtree(extracted_dir) logger.info( - "All zips merged! Total copied: %s, Total overwritten: %s, Total processed: %s", + "All zips merged! Total copied: %s, Total duplicated: %s, Total processed: %s", total_copied, - total_overwritten, + total_duplicated, total_processed, ) return final_tmp_root @@ -743,10 +743,10 @@ def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): * src_dir is the source directory to copy from * dest_dir is the destination directory to copy to. - Returns Tuple of (copied_count, overwritten_count, processed_count) + Returns Tuple of (copied_count, duplicated_count, processed_count) """ copied_count = 0 - overwritten_count = 0 + duplicated_count = 0 processed_count = 0 # Find the actual content directory @@ -768,27 +768,27 @@ def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): # Create destination directory if it doesn't exist os.makedirs(dest_root, exist_ok=True) - # Copy all files, overwriting existing ones + # Copy all files, skip existing ones for file in files: src_file = os.path.join(root_dir, file) dest_file = os.path.join(dest_root, file) if os.path.exists(dest_file): - overwritten_count += 1 - logger.debug("Overwritten: %s -> %s", src_file, dest_file) + duplicated_count += 1 + logger.debug("Duplicated: %s, skipped", dest_file) else: copied_count += 1 + copy2(src_file, dest_file) logger.debug("Copied: %s -> %s", src_file, dest_file) processed_count += 1 - copy2(src_file, dest_file) logger.info( - "One zip merged! Files copied: %s, Files overwritten: %s, Total files processed: %s", + "One zip merged! Files copied: %s, Files duplicated: %s, Total files processed: %s", copied_count, - overwritten_count, + duplicated_count, processed_count, ) - return copied_count, overwritten_count, processed_count + return copied_count, duplicated_count, processed_count def _scan_paths(files_root: str, ignore_patterns: List[str], From cd7ed7a10cde676307e45ee7e8cbae0e8fe92131 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 22 Oct 2025 15:03:01 +0800 Subject: [PATCH 53/64] Add archetype catalog files merged logic for merged zips --- charon/pkgs/maven.py | 89 +++++++++++++++++++++++++++++++++++--- tests/test_maven_upload.py | 1 + 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 6ac406c5..1cf23041 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -690,6 +690,7 @@ def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str total_copied = 0 total_duplicated = 0 + total_merged = 0 total_processed = 0 # Collect all extracted directories first @@ -719,20 +720,23 @@ def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str # Merge content from all extracted directories for extracted_dir in extracted_dirs: - copied, duplicated, processed = _merge_directories_with_rename( + copied, duplicated, merged, processed = _merge_directories_with_rename( extracted_dir, merged_dest_dir, root ) total_copied += copied total_duplicated += duplicated + total_merged += merged total_processed += processed # Clean up temporary extraction directory rmtree(extracted_dir) logger.info( - "All zips merged! Total copied: %s, Total duplicated: %s, Total processed: %s", + "All zips merged! Total copied: %s, Total duplicated: %s, " + "Total merged: %s, Total processed: %s", total_copied, total_duplicated, + total_merged, total_processed, ) return final_tmp_root @@ -743,10 +747,11 @@ def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): * src_dir is the source directory to copy from * dest_dir is the destination directory to copy to. - Returns Tuple of (copied_count, duplicated_count, processed_count) + Returns Tuple of (copied_count, duplicated_count, merged_count, processed_count) """ copied_count = 0 duplicated_count = 0 + merged_count = 0 processed_count = 0 # Find the actual content directory @@ -772,23 +777,95 @@ def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): for file in files: src_file = os.path.join(root_dir, file) dest_file = os.path.join(dest_root, file) + + if file == ARCHETYPE_CATALOG_FILENAME: + _handle_archetype_catalog_merge(src_file, dest_file) + merged_count += 1 + logger.debug("Merged archetype catalog: %s -> %s", src_file, dest_file) if os.path.exists(dest_file): duplicated_count += 1 logger.debug("Duplicated: %s, skipped", dest_file) else: - copied_count += 1 copy2(src_file, dest_file) + copied_count += 1 logger.debug("Copied: %s -> %s", src_file, dest_file) processed_count += 1 logger.info( - "One zip merged! Files copied: %s, Files duplicated: %s, Total files processed: %s", + "One zip merged! Files copied: %s, Files duplicated: %s, " + "Files merged: %s, Total files processed: %s", copied_count, duplicated_count, + merged_count, processed_count, ) - return copied_count, duplicated_count, processed_count + return copied_count, duplicated_count, merged_count, processed_count + + +def _handle_archetype_catalog_merge(src_catalog: str, dest_catalog: str): + """ + Handle merging of archetype-catalog.xml files during directory merge. + + Args: + src_catalog: Source archetype-catalog.xml file path + dest_catalog: Destination archetype-catalog.xml file path + """ + try: + with open(src_catalog, "rb") as sf: + src_archetypes = _parse_archetypes(sf.read()) + except ElementTree.ParseError as e: + logger.warning("Failed to read source archetype catalog %s: %s", src_catalog, e) + return + + if len(src_archetypes) < 1: + logger.warning( + "No archetypes found in source archetype-catalog.xml: %s, " + "even though the file exists! Skipping.", + src_catalog + ) + return + + # Copy directly if dest_catalog doesn't exist + if not os.path.exists(dest_catalog): + copy2(src_catalog, dest_catalog) + return + + try: + with open(dest_catalog, "rb") as df: + dest_archetypes = _parse_archetypes(df.read()) + except ElementTree.ParseError as e: + logger.warning("Failed to read dest archetype catalog %s: %s", dest_catalog, e) + return + + if len(dest_archetypes) < 1: + logger.warning( + "No archetypes found in dest archetype-catalog.xml: %s, " + "even though the file exists! Copy directly from the src_catalog, %s.", + dest_catalog, src_catalog + ) + copy2(src_catalog, dest_catalog) + return + + else: + original_dest_size = len(dest_archetypes) + for sa in src_archetypes: + if sa not in dest_archetypes: + dest_archetypes.append(sa) + else: + logger.debug("DUPLICATE ARCHETYPE: %s", sa) + + if len(dest_archetypes) != original_dest_size: + with open(dest_catalog, 'wb'): + content = MavenArchetypeCatalog(dest_archetypes).generate_meta_file_content() + try: + overwrite_file(dest_catalog, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + dest_catalog, + ) + raise e def _scan_paths(files_root: str, ignore_patterns: List[str], diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index fefa74ea..ab36c76f 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -168,6 +168,7 @@ def test_multi_zips_upload(self): catalog = self.test_bucket.Object(ARCHETYPE_CATALOG) cat_content = str(catalog.get()["Body"].read(), "utf-8") + self.assertIn("4.5.6", cat_content) self.assertIn("4.5.9", cat_content) self.assertIn("httpclient", cat_content) self.assertIn("org.apache.httpcomponents", cat_content) From 56c5a33b09ef47bf8c1aed2c13279c4ea5c4a187 Mon Sep 17 00:00:00 2001 From: yma Date: Thu, 23 Oct 2025 09:27:07 +0800 Subject: [PATCH 54/64] Fix unnecessary file open during archetype catalog merge --- charon/pkgs/maven.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 1cf23041..25f8bc4f 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -856,16 +856,12 @@ def _handle_archetype_catalog_merge(src_catalog: str, dest_catalog: str): logger.debug("DUPLICATE ARCHETYPE: %s", sa) if len(dest_archetypes) != original_dest_size: - with open(dest_catalog, 'wb'): - content = MavenArchetypeCatalog(dest_archetypes).generate_meta_file_content() - try: - overwrite_file(dest_catalog, content) - except FileNotFoundError as e: - logger.error( - "Error: Can not create file %s because of some missing folders", - dest_catalog, - ) - raise e + content = MavenArchetypeCatalog(dest_archetypes).generate_meta_file_content() + try: + overwrite_file(dest_catalog, content) + except Exception as e: + logger.error("Failed to merge archetype catalog: %s", dest_catalog) + raise e def _scan_paths(files_root: str, ignore_patterns: List[str], From 9afe959ba70641d09465bf1434faa44d9510be3c Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 23 Oct 2025 14:23:22 +0800 Subject: [PATCH 55/64] Fix: remove two redundant open operations --- charon/pkgs/maven.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 25f8bc4f..02692e80 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -1032,17 +1032,16 @@ def _generate_rollback_archetype_catalog( else: # Re-render the result of our archetype un-merge to the # local file, in preparation for upload. - with open(local, 'wb') as f: - content = MavenArchetypeCatalog(remote_archetypes)\ - .generate_meta_file_content() - try: - overwrite_file(local, content) - except FileNotFoundError as e: - logger.error( - "Error: Can not create file %s because of some missing folders", - local, - ) - raise e + content = MavenArchetypeCatalog(remote_archetypes)\ + .generate_meta_file_content() + try: + overwrite_file(local, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + local, + ) + raise e __gen_all_digest_files(local) return 1 @@ -1148,17 +1147,16 @@ def _generate_upload_archetype_catalog( # Re-render the result of our archetype merge / # un-merge to the local file, in preparation for # upload. - with open(local, 'wb') as f: - content = MavenArchetypeCatalog(remote_archetypes)\ - .generate_meta_file_content() - try: - overwrite_file(local, content) - except FileNotFoundError as e: - logger.error( - "Error: Can not create file %s because of some missing folders", - local, - ) - raise e + content = MavenArchetypeCatalog(remote_archetypes)\ + .generate_meta_file_content() + try: + overwrite_file(local, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + local, + ) + raise e __gen_all_digest_files(local) return True From 46cf0c55897a947e39e7f7746bacfb71fb36e0ba Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 29 Oct 2025 09:07:59 +0800 Subject: [PATCH 56/64] Update version to 1.3.5 --- charon.spec | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charon.spec b/charon.spec index 95eb21a9..c6e38705 100644 --- a/charon.spec +++ b/charon.spec @@ -1,7 +1,7 @@ %global owner Commonjava %global modulename charon -%global charon_version 1.3.4 +%global charon_version 1.3.5 %global sdist_tar_name %{modulename}-%{charon_version} %global python3_pkgversion 3 diff --git a/pyproject.toml b/pyproject.toml index cd439657..c0735696 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["setuptools", "setuptools-scm"] [project] name = "charon" -version = "1.3.4" +version = "1.3.5" authors = [ {name = "RedHat EXD SPMM"}, ] diff --git a/setup.py b/setup.py index 5c37fc84..eb9b51fb 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ """ from setuptools import setup, find_packages -version = "1.3.4" +version = "1.3.5" long_description = """ This charon is a tool to synchronize several types of From 57f74132525505d314b96fb1f66cd739f6966722 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 29 Oct 2025 09:39:34 +0800 Subject: [PATCH 57/64] Update charon.spec for release notes --- charon.spec | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/charon.spec b/charon.spec index c6e38705..4e68547b 100644 --- a/charon.spec +++ b/charon.spec @@ -64,6 +64,25 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8 %changelog +* Wed Oct 29 2025 Gang Li +- 1.3.5 release +- Support recursive indexing for index function +- Accept multiple maven zips for uploading + +* Mon Jun 23 2025 Gang Li +- 1.3.4 release +- Add --version flag to support version check +- Bug fix: MMENG-4362 re-sort the indexing page items +- Add pyproject.toml + +* Mon Dec 16 2024 Gang Li +- 1.3.3 release +- Bug fix: MMENG-4284 npm del error when deleting a package which has overlapped name with others + +* Wed Jul 10 2024 Gang Li +- 1.3.2 release +- Container file update + * Tue May 7 2024 Gang Li - 1.3.1 release - Add checksum refresh command: refresh checksum files for maven artifacts From 1dad8793da6c1f06cf8a96fae1a3da0113d97bb0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 29 Oct 2025 09:51:50 +0800 Subject: [PATCH 58/64] Update README --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c0cb8377..7b9e9cd3 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,14 @@ to configure AWS access credentials. ### charon-upload: upload a repo to S3 ```bash -usage: charon upload $tarball --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--contain_signature] [--key] +usage: charon upload $tarball [$tarball*] --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--contain_signature] [--key] ``` This command will upload the repo in tarball to S3. It will auto-detect if the tarball is for maven or npm +**New in 1.3.5**: For Maven archives, this command now supports uploading multiple zip files at once. When multiple Maven zips are provided, they will be merged intelligently, including proper handling of archetype catalog files and duplicate artifact detection. + * For maven type, it will: * Scan the tarball for all paths and collect them all. @@ -99,11 +101,13 @@ This command will delete some paths from repo in S3. ### charon-index: refresh the index.html for the specified path ```bash -usage: charon index $PATH [-t, --target] [-D, --debug] [-q, --quiet] +usage: charon index $PATH [-t, --target] [-D, --debug] [-q, --quiet] [--recursive] ``` This command will refresh the index.html for the specified path. +**New in 1.3.5**: Added `--recursive` flag to support recursive indexing under the specified path. + * Note that if the path is a NPM metadata path which contains package.json, this refreshment will not work because this type of folder will display the package.json instead of the index.html in http request. ### charon-cf-check: check the invalidation status of the specified invalidation id for AWS CloudFront From 9113f1eeda3f8137617be88e3e1365281158d3b5 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 26 Nov 2025 17:08:03 +0800 Subject: [PATCH 59/64] Fix some potential security issues Assisted by: Claude code --- charon/pkgs/maven.py | 68 ++++++++++++++++++++++++++++++------------- charon/utils/files.py | 52 +++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 43 deletions(-) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 02692e80..070a3101 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -32,7 +32,7 @@ META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME, PACKAGE_TYPE_MAVEN) -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Union from jinja2 import Template from datetime import datetime from zipfile import ZipFile, BadZipFile @@ -217,7 +217,8 @@ def parse_gavs(pom_paths: List[str], root="/") -> Dict[str, Dict[str, List[str]] return gavs -def gen_meta_file(group_id, artifact_id: str, versions: list, root="/", digest=True) -> List[str]: +def gen_meta_file(group_id, artifact_id: str, + versions: list, root="/", do_digest=True) -> List[str]: content = MavenMetadata( group_id, artifact_id, versions ).generate_meta_file_content() @@ -229,7 +230,7 @@ def gen_meta_file(group_id, artifact_id: str, versions: list, root="/", digest=T meta_files.append(final_meta_path) except FileNotFoundError as e: raise e - if digest: + if do_digest: meta_files.extend(__gen_all_digest_files(final_meta_path)) return meta_files @@ -782,7 +783,7 @@ def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): _handle_archetype_catalog_merge(src_file, dest_file) merged_count += 1 logger.debug("Merged archetype catalog: %s -> %s", src_file, dest_file) - if os.path.exists(dest_file): + elif os.path.exists(dest_file): duplicated_count += 1 logger.debug("Duplicated: %s, skipped", dest_file) else: @@ -1303,8 +1304,8 @@ def __wildcard_metadata_paths(paths: List[str]) -> List[str]: new_paths.append(path[:-len(".xml")] + ".*") elif path.endswith(".md5")\ or path.endswith(".sha1")\ - or path.endswith(".sha128")\ - or path.endswith(".sha256"): + or path.endswith(".sha256")\ + or path.endswith(".sha512"): continue else: new_paths.append(path) @@ -1313,7 +1314,7 @@ def __wildcard_metadata_paths(paths: List[str]) -> List[str]: class VersionCompareKey: 'Used as key function for version sorting' - def __init__(self, obj): + def __init__(self, obj: str): self.obj = obj def __lt__(self, other): @@ -1344,36 +1345,61 @@ def __compare(self, other) -> int: big = max(len(xitems), len(yitems)) for i in range(big): try: - xitem = xitems[i] + xitem: Union[str, int] = xitems[i] except IndexError: return -1 try: - yitem = yitems[i] + yitem: Union[str, int] = yitems[i] except IndexError: return 1 - if xitem.isnumeric() and yitem.isnumeric(): + if (isinstance(xitem, str) and isinstance(yitem, str) and + xitem.isnumeric() and yitem.isnumeric()): xitem = int(xitem) yitem = int(yitem) - elif xitem.isnumeric() and not yitem.isnumeric(): + elif (isinstance(xitem, str) and xitem.isnumeric() and + (not isinstance(yitem, str) or not yitem.isnumeric())): return 1 - elif not xitem.isnumeric() and yitem.isnumeric(): - return -1 - if xitem > yitem: - return 1 - elif xitem < yitem: + elif (isinstance(yitem, str) and yitem.isnumeric() and + (not isinstance(xitem, str) or not xitem.isnumeric())): return -1 + # At this point, both are the same type (both int or both str) + if isinstance(xitem, int) and isinstance(yitem, int): + if xitem > yitem: + return 1 + elif xitem < yitem: + return -1 + elif isinstance(xitem, str) and isinstance(yitem, str): + if xitem > yitem: + return 1 + elif xitem < yitem: + return -1 else: continue return 0 -class ArchetypeCompareKey(VersionCompareKey): - 'Used as key function for GAV sorting' - def __init__(self, gav): - super().__init__(gav.version) +class ArchetypeCompareKey: + def __init__(self, gav: ArchetypeRef): self.gav = gav - # pylint: disable=unused-private-member + def __lt__(self, other): + return self.__compare(other) < 0 + + def __gt__(self, other): + return self.__compare(other) > 0 + + def __le__(self, other): + return self.__compare(other) <= 0 + + def __ge__(self, other): + return self.__compare(other) >= 0 + + def __eq__(self, other): + return self.__compare(other) == 0 + + def __hash__(self): + return self.gav.__hash__() + def __compare(self, other) -> int: x = self.gav.group_id + ":" + self.gav.artifact_id y = other.gav.group_id + ":" + other.gav.artifact_id diff --git a/charon/utils/files.py b/charon/utils/files.py index ccad3e23..dca71444 100644 --- a/charon/utils/files.py +++ b/charon/utils/files.py @@ -17,7 +17,9 @@ import os import hashlib import errno -from typing import List, Tuple +import tempfile +import shutil +from typing import List, Tuple, Optional from charon.constants import MANIFEST_SUFFIX @@ -32,24 +34,37 @@ class HashType(Enum): def get_hash_type(type_str: str) -> HashType: """Get hash type from string""" - if type_str.lower() == "md5": + type_str_low = type_str.lower() + if type_str_low == "md5": return HashType.MD5 - elif type_str.lower() == "sha1": + elif type_str_low == "sha1": return HashType.SHA1 - elif type_str.lower() == "sha256": + elif type_str_low == "sha256": return HashType.SHA256 - elif type_str.lower() == "sha512": + elif type_str_low == "sha512": return HashType.SHA512 else: raise ValueError("Unsupported hash type: {}".format(type_str)) -def overwrite_file(file_path: str, content: str): - if not os.path.isfile(file_path): - with open(file_path, mode="a", encoding="utf-8"): - pass - with open(file_path, mode="w", encoding="utf-8") as f: - f.write(content) +def overwrite_file(file_path: str, content: str) -> None: + parent_dir: Optional[str] = os.path.dirname(file_path) + if parent_dir: + if not os.path.exists(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + else: + parent_dir = None # None explicitly means current directory for tempfile + + # Write to temporary file first, then atomically rename + fd, temp_path = tempfile.mkstemp(dir=parent_dir, text=True) + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + f.write(content) + shutil.move(temp_path, file_path) + except Exception: + if os.path.exists(temp_path): + os.unlink(temp_path) + raise def read_sha1(file: str) -> str: @@ -97,7 +112,6 @@ def digest_content(content: str, hash_type=HashType.SHA1) -> str: def _hash_object(hash_type: HashType): - hash_obj = None if hash_type == HashType.SHA1: hash_obj = hashlib.sha1() elif hash_type == HashType.SHA256: @@ -107,7 +121,7 @@ def _hash_object(hash_type: HashType): elif hash_type == HashType.SHA512: hash_obj = hashlib.sha512() else: - raise Exception("Error: Unknown hash type for digesting.") + raise ValueError("Error: Unknown hash type for digesting.") return hash_obj @@ -116,14 +130,8 @@ def write_manifest(paths: List[str], root: str, product_key: str) -> Tuple[str, manifest_path = os.path.join(root, manifest_name) artifacts = [] for path in paths: - if path.startswith(root): - path = path[len(root):] - if path.startswith("/"): - path = path[1:] - artifacts.append(path) - - if not os.path.isfile(manifest_path): - with open(manifest_path, mode="a", encoding="utf-8"): - pass + rel_path = os.path.relpath(path, root) + artifacts.append(rel_path) + overwrite_file(manifest_path, '\n'.join(artifacts)) return manifest_name, manifest_path From 4a96f1f53b73a33f269bd7672ac9cf767c61add0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Sat, 29 Nov 2025 19:34:45 +0800 Subject: [PATCH 60/64] Fix tests issue with new merged code from 1.3.x The new code from 1.3.x fixed some potential issue of file writting in files.overwrite_file, however it breaks the tests of radas. Assisted by: Claude code --- tests/test_radas_sign_generation.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/test_radas_sign_generation.py b/tests/test_radas_sign_generation.py index ccc448a2..8e64d76c 100644 --- a/tests/test_radas_sign_generation.py +++ b/tests/test_radas_sign_generation.py @@ -20,7 +20,6 @@ import os import json import shutil -import builtins from unittest import mock from charon.utils.files import overwrite_file from charon.pkgs.radas_sign import generate_radas_sign @@ -68,20 +67,14 @@ def test_sign_files_generation_with_failure(self): expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") - # simulate expected_asc1 can not open to write properly - real_open = builtins.open - with mock.patch("builtins.open") as mock_open: - def side_effect(path, *args, **kwargs): - # this is for pylint check - mode = "r" - if len(args) > 0: - mode = args[0] - elif "mode" in kwargs: - mode = kwargs["mode"] - if path == expected_asc1 and "w" in mode: + # simulate expected_asc1 can not be written properly + real_overwrite = overwrite_file + with mock.patch("charon.pkgs.radas_sign.files.overwrite_file") as mock_overwrite: + def side_effect(path, content): + if path == expected_asc1: raise IOError("mock write error") - return real_open(path, *args, **kwargs) - mock_open.side_effect = side_effect + return real_overwrite(path, content) + mock_overwrite.side_effect = side_effect failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(len(failed), 1) From 10ed3a8c878096751ef3b8385bbc1369ee20c039 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 19 Nov 2025 09:16:14 +0800 Subject: [PATCH 61/64] Feat: Add merge cmd for multiple maven zips --- charon/cmd/__init__.py | 4 ++ charon/cmd/cmd_merge.py | 154 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 charon/cmd/cmd_merge.py diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index 985d7f79..f3d027b9 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -20,6 +20,7 @@ from charon.cmd.cmd_checksum import init_checksum, checksum from charon.cmd.cmd_cache import init_cf, cf from charon.cmd.cmd_sign import sign +from charon.cmd.cmd_merge import merge @group() @@ -47,3 +48,6 @@ def cli(ctx): # radas sign cmd cli.add_command(sign) + +# maven zips merge cmd +cli.add_command(merge) diff --git a/charon/cmd/cmd_merge.py b/charon/cmd/cmd_merge.py new file mode 100644 index 00000000..84b344b3 --- /dev/null +++ b/charon/cmd/cmd_merge.py @@ -0,0 +1,154 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List + +from charon.utils.archive import detect_npm_archives, NpmArchiveType +from charon.cmd.internal import _get_local_repos, _decide_mode +from charon.pkgs.maven import _extract_tarballs +from click import command, option, argument +from zipfile import ZipFile, ZIP_DEFLATED +from tempfile import mkdtemp + +import logging +import os +import sys + +logger = logging.getLogger(__name__) + + +@argument( + "repos", + type=str, + nargs=-1 # This allows multiple arguments for zip urls +) +@option( + "--product", + "-p", + help=""" + The product key, will combine with version to decide + the metadata of the files in tarball. + """, + nargs=1, + required=True, + multiple=False, +) +@option( + "--version", + "-v", + help=""" + The product version, will combine with key to decide + the metadata of the files in tarball. + """, + required=True, + multiple=False, +) +@option( + "--root_path", + "-r", + default="maven-repository", + help=""" + The root path in the tarball before the real maven paths, + will be trailing off before uploading. + """, +) +@option( + "--work_dir", + "-w", + help=""" + The temporary working directory into which archives should + be extracted, when needed. + """, +) +@option( + "--merge_result", + "-m", + help=""" + The path of the final merged zip file will be compressed and saved. + """, +) +@option( + "--debug", + "-D", + help="Debug mode, will print all debug logs for problem tracking.", + is_flag=True, + default=False +) +@option( + "--quiet", + "-q", + help="Quiet mode, will shrink most of the logs except warning and errors.", + is_flag=True, + default=False +) +@command() +def merge( + repos: List[str], + product: str, + version: str, + root_path="maven-repository", + work_dir=None, + merge_result=None, + debug=False, + quiet=False +): + """Merge multiple Maven ZIP archives and compress the result into a single ZIP file. + The merged file is stored locally as specified by merge_result. + + Note: This function does not support merging single archive, NPM archives, + or archives of inconsistent types. + """ + _decide_mode(product, version, is_quiet=quiet, is_debug=debug) + if len(repos) == 1: + logger.info("Skip merge step, single archive detected, no merge needed") + sys.exit(0) + + product_key = f"{product}-{version}" + archive_paths = _get_local_repos(repos) + archive_types = detect_npm_archives(archive_paths) + + maven_count = archive_types.count(NpmArchiveType.NOT_NPM) + npm_count = len(archive_types) - maven_count + if maven_count == len(archive_types): + tmp_root = _extract_tarballs(archive_paths, root_path, product_key, dir__=work_dir) + _create_merged_zip(tmp_root, merge_result, product_key, work_dir) + elif npm_count == len(archive_types): + logger.error("Skip merge step for the npm archives") + sys.exit(1) + else: + logger.error("Skip merge step since the types are not consistent") + sys.exit(1) + + +def _create_merged_zip( + root_path: str, + merge_result: str, + product_key: str, + work_dir: str +): + zip_path = merge_result + if not merge_result: + merge_path = mkdtemp(prefix=f"{product_key}_merged_", dir=work_dir) + zip_path = os.path.join(merge_path, f"{product_key}_merged.zip") + + # pylint: disable=unused-variable + with ZipFile(zip_path, 'w', ZIP_DEFLATED) as zipf: + for root, dirs, files in os.walk(root_path): + for file in files: + file_path = os.path.join(root, file) + # Calculate relative path to preserve directory structure + arcname = os.path.relpath(file_path, root_path) + zipf.write(file_path, arcname) + logger.info("Done for the merged zip generation: %s", zip_path) From 0a32943e216af06f3dd0bc3cd9651c59b2ac3530 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 26 Nov 2025 17:21:53 +0800 Subject: [PATCH 62/64] Update the param merge_result note about default value description --- charon/cmd/cmd_merge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charon/cmd/cmd_merge.py b/charon/cmd/cmd_merge.py index 84b344b3..3371ef74 100644 --- a/charon/cmd/cmd_merge.py +++ b/charon/cmd/cmd_merge.py @@ -77,6 +77,8 @@ "-m", help=""" The path of the final merged zip file will be compressed and saved. + Default is the ZIP file which is created in a temporary directory based on work_dir. + e.g. /tmp/work/jboss-eap-8.1.0_merged_a1b2c3/jboss-eap-8.1.0_merged.zip """, ) @option( From d896e27d596091c3887382cc7b353935338b83d6 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 1 Dec 2025 12:26:04 +0800 Subject: [PATCH 63/64] Upgrade pytest to 9.0.1 --- pyproject.toml | 2 +- tests/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4f87614b..0693668d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dev = [ test = [ "flexmock>=0.10.6", "responses>=0.9.0", - "pytest<=8.4.1", + "pytest<=9.0.1", "pytest-cov", "pytest-html", "requests-mock", diff --git a/tests/requirements.txt b/tests/requirements.txt index 477e546b..b0e35b7f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ flexmock>=0.10.6 responses>=0.9.0 -pytest<=8.4.1 +pytest<=9.0.1 pytest-cov pytest-html requests-mock From 392ab6f78081d923bad4b98ca619e7ee185d1ed0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 1 Dec 2025 13:36:06 +0800 Subject: [PATCH 64/64] Fix a spec file issue Seems the changelog is using date decending order --- charon.spec | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charon.spec b/charon.spec index c8219cf4..d1ea5170 100644 --- a/charon.spec +++ b/charon.spec @@ -64,15 +64,15 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8 %changelog -* Fri Jun 27 2025 Gang Li -- 1.4.0 release -- Add RADAS signature support - * Wed Oct 29 2025 Gang Li - 1.3.5 release - Support recursive indexing for index function - Accept multiple maven zips for uploading +* Fri Jun 27 2025 Gang Li +- 1.4.0 release +- Add RADAS signature support + * Mon Jun 23 2025 Gang Li - 1.3.4 release - Add --version flag to support version check