diff --git a/docs/source/tutorials/automated-publication-with-ci.md b/docs/source/tutorials/automated-publication-with-ci.md index 172cb7df..ce3d7417 100644 --- a/docs/source/tutorials/automated-publication-with-ci.md +++ b/docs/source/tutorials/automated-publication-with-ci.md @@ -46,6 +46,10 @@ This also works with many Jupyter Hubs, no need for a Linux computer! ```{code-block} bash python -m pip install hermes ``` + If already installed, make sure you have the latest version by running: + ```{code-block} bash + python -m pip install hermes --upgrade + ``` 5. Once installed, navigate to the main directory of your project. ```{code-block} bash cd myproject/ diff --git a/poetry.lock b/poetry.lock index 36e3277f..04086bb1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -764,7 +764,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["docs"] +groups = ["main", "docs"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -1018,7 +1018,7 @@ version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["docs"] +groups = ["main", "docs"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -2636,4 +2636,4 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0.0" -content-hash = "0cfba018c4947b865c5fadc25c473e2aad1704a866e5242524b1c50e8f1f0d88" +content-hash = "c74e2079ece81c4a388a52f1027c7a846e5c3beabd983882db13d82bb8b88889" diff --git a/pyproject.toml b/pyproject.toml index 2457db8a..61758dd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "pydantic-settings>=2.1.0, <3.0.0", "requests-oauthlib>=2.0.0, <3.0.0", "pynacl>=1.6.2, <2.0.0", + "jinja2 (>=3.1.6,<4.0.0)", ] requires-python = ">=3.11, <4.0.0" @@ -48,12 +49,10 @@ documentation = "https://hermes.software-metadata.pub" repository = "https://github.com/softwarepub/hermes.git" issues = "https://github.com/softwarepub/hermes/issues" - [project.scripts] hermes = "hermes.commands.cli:main" hermes-marketplace = "hermes.commands.marketplace:main" - [project.entry-points."hermes.harvest"] cff = "hermes.commands.harvest.cff:CffHarvestPlugin" codemeta = "hermes.commands.harvest.codemeta:CodeMetaHarvestPlugin" @@ -106,14 +105,12 @@ reuse = "^1.1.2" sphinxcontrib-datatemplates = "~=0.11" - [tool.taskipy.tasks] docs-build = "poetry run sphinx-build -M html docs/source docs/build -W" docs-clean = "poetry run sphinx-build -M clean docs/source docs/build" docs-live = "poetry run sphinx-autobuild docs/source docs/build" flake8 = "poetry run flake8 ./test/ ./src/ --count --select=E9,F63,F7,F82 --statistics" - [tool.pytest.ini_options] norecursedirs = "docs/*" testpaths = [ @@ -121,7 +118,6 @@ testpaths = [ ] addopts = "--cov=hermes --cov-report term" - [build-system] requires = [ "poetry-core>=2.1.3, <3.0.0" diff --git a/src/hermes/commands/deposit/invenio.py b/src/hermes/commands/deposit/invenio.py index 703e9a5b..7d8e4c0a 100644 --- a/src/hermes/commands/deposit/invenio.py +++ b/src/hermes/commands/deposit/invenio.py @@ -267,21 +267,6 @@ def __init__(self, command: HermesDepositCommand, ctx: CodeMetaContext, client=N if client is None: auth_token = self.config.auth_token - - # TODO reactivate this code again, once we use Zenodo OAuth again (once the refresh token works) - # If auth_token is a refresh-token, get the auth-token from that. - # if str(auth_token).startswith("REFRESH_TOKEN:"): - # _log.debug(f"Getting token from refresh_token {auth_token}") - # # TODO How do we know if this targets sandbox or not? - # # Now we assume it's sandbox - # connect_zenodo.setup(True) - # tokens = connect_zenodo.oauth_process() \ - # .get_tokens_from_refresh_token(auth_token.split("REFRESH_TOKEN:")[1]) - # _log.debug(f"Tokens: {str(tokens)}") - # auth_token = tokens.get("access_token", "") - # _log.debug(f"Auth Token: {auth_token}") - # # TODO Update the secret (github/lab token is needed) - if not auth_token: raise DepositionUnauthorizedError("No valid auth token given for deposition platform") self.client = self.invenio_client_class(self.config, diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index c3f61cc8..f77f3c77 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -15,6 +15,8 @@ from pathlib import Path from urllib.parse import urljoin, urlparse +import jinja2 +import jinja2.meta import requests import toml from pydantic import BaseModel @@ -27,6 +29,7 @@ connect_zenodo, git_info) TUTORIAL_URL = "https://hermes.software-metadata.pub/en/latest/tutorials/automated-publication-with-ci.html" +REPOSITORY_URL = "https://github.com/softwarepub/hermes" class GitHoster(Enum): @@ -35,22 +38,50 @@ class GitHoster(Enum): GitLab = auto() -class DepositPlatform(Enum): +class DepositId(Enum): + """ + Enum as additional identifier for DepositPlatforms so we have something persistent to code with. + """ Empty = auto() Zenodo = auto() ZenodoSandbox = auto() + # JuelichData = auto() + # JuelichDataBeta = auto() + # DemoDataverse = auto() + Rodare = auto() + RodareTest = auto() -DepositPlatformNames: dict[DepositPlatform, str] = { - DepositPlatform.ZenodoSandbox: "Zenodo (Sandbox)", - DepositPlatform.Zenodo: "Zenodo", - } - - -DepositPlatformUrls: dict[DepositPlatform, str] = { - DepositPlatform.Zenodo: "https://zenodo.org/", - DepositPlatform.ZenodoSandbox: "https://sandbox.zenodo.org/" - } +@dataclass +class DepositPlatform: + """ + This dataclass contains all relevant data to set up hermes for a given platform. + """ + def __init__(self, name: str = "", url: str = "", plugin_name: str = "", deposit_id: DepositId = DepositId.Empty): + self.name: str = name + self.url: str = url + """Base url of the deposit platform""" + self.plugin_name: str = plugin_name + """Internal name of our related hermes deposit plugin""" + self.id: DepositId = deposit_id + """Non changing enum-based ID to keep the consistency""" + self.internal_name: str = deposit_id.name + self.token: str = "" + """This is the access token which will get filled in connect_deposit_platform""" + self.token_name: str = re.sub(r'(? GitHoster: def download_file_from_url(url, filepath, append: bool = False) -> None: + if not append and os.path.exists(filepath): + os.remove(filepath) try: with requests.get(url, stream=True) as r: r.raise_for_status() @@ -123,7 +156,7 @@ def string_in_file(file_path, search_string: str) -> bool: return any(search_string in line for line in file) -def get_builtin_plugins(plugin_commands: list[str]) -> dict[str: HermesPlugin]: +def get_builtin_plugins(plugin_commands: list[str]) -> dict[str, HermesPlugin]: """ Returns a list of installed HermesPlugins based on a list of related command names. This is currently not used (we use the marketplace code instead) but maybe later. @@ -163,43 +196,51 @@ def __init__(self, parser: argparse.ArgumentParser): super().__init__(parser) self.folder_info: HermesInitFolderInfo = HermesInitFolderInfo() self.hermes_was_already_installed: bool = False + self.warn_on_old_version: bool = True self.new_created_paths: list[Path] = [] self.tokens: dict = {} self.setup_method: str = "" - self.deposit_platform: DepositPlatform = DepositPlatform.Empty + self.deposit_platform: DepositPlatform = DepositPlatform() + self.git_branch: str = "" self.git_remote: str = "" self.git_remote_url = "" self.git_hoster: GitHoster = GitHoster.Empty self.template_base_url: str = "https://raw.githubusercontent.com" - self.template_branch: str = "feature/init-custom-ci" + self.template_branch: str = "main" self.template_repo: str = "softwarepub/ci-templates" - self.template_folder: str = "init" + self.template_folder: str = "init-templates" self.ci_parameters: dict = { + "pip_install_hermes": "pip install hermes", + "pip_install_plugins_github": "", + "pip_install_plugins_gitlab": "", "deposit_zip_name": "artifact.zip", "deposit_zip_files": "", "deposit_initial": "--initial", - "deposit_extra_files": "", - "push_branch": "main" + "deposit_extra_files": "--file ???", + "deposit_parameter_token": "-O ???.auth_token", + "deposit_parameter_zip_file": "--file ???.zip", + "deposit_token_name": "???_TOKEN", + "gh_push_branches_or_tags": "branches", + "gh_push_target": "main", + "gl_push_condition": "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" } self.hermes_toml_data = { "harvest": { "sources": ["cff"] }, "deposit": { - "target": "invenio_rdm", - "invenio_rdm": { - "site_url": "", - "access_right": "open" - } + "target": "", } } self.plugin_relevant_commands = ["harvest", "deposit"] - self.builtin_plugins: dict[str: HermesPlugin] = get_builtin_plugins(self.plugin_relevant_commands) + self.builtin_plugins: dict[str, HermesPlugin] = get_builtin_plugins(self.plugin_relevant_commands) self.selected_plugins: list[marketplace.PluginInfo] = [] def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: command_parser.add_argument('--template-branch', nargs=1, default="", help="Branch or tag of the ci-templates repository.") + command_parser.add_argument('--hermes-branch', nargs=1, default="", + help="Branch of the hermes repository which will be used in the pipeline.") def load_settings(self, args: argparse.Namespace): pass @@ -224,10 +265,20 @@ def __call__(self, args: argparse.Namespace) -> None: # Setup logging self.setup_file_logging() - # Save command parameter (template branch) + # Warning on old hermes version + self.check_hermes_version() + + # Process command parameters (ci-templates branch & hermes branch) if hasattr(args, "template_branch"): if args.template_branch != "": self.template_branch = args.template_branch + if hasattr(args, "hermes_branch"): + if args.hermes_branch: + branch_name = args.hermes_branch[0] + if branch_name != "": + sc.echo(f"Using Hermes branch: {branch_name}") + self.ci_parameters["pip_install_hermes"] = \ + f"pip install git+{REPOSITORY_URL}.git@{branch_name}" try: # Test if init is valid in current folder @@ -246,7 +297,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.choose_setup_method() sc.next_step("Configure HERMES behaviour") - self.choose_push_branch() + self.choose_push_trigger() self.choose_deposit_files() sc.next_step("Create hermes.toml file") @@ -266,7 +317,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.configure_git_project() self.clean_up_files(False) - sc.echo("\nHERMES is now initialized and ready to be used.\n", + sc.echo("\nHERMES is now initialized. Add the changes to your git index and it is ready to be used.\n", formatting=sc.Formats.OKGREEN+sc.Formats.BOLD) # Nice message on Ctrl+C @@ -282,8 +333,37 @@ def __call__(self, args: argparse.Namespace) -> None: formatting=sc.Formats.FAIL+sc.Formats.BOLD) sc.debug_info(traceback.format_exc()) self.clean_up_files(True) + sc.echo("The initialization was not finalized. You will have to run 'hermes init' again.") sys.exit(2) + def check_hermes_version(self) -> None: + """Fetches the current Pypi Hermes version. Gives a warning if the current version is not up to date.""" + if not self.warn_on_old_version: + return + try: + current_hermes_version: str = metadata.version("hermes") + pypi_hermes_json: dict = requests.get("https://pypi.org/pypi/hermes/json", timeout=10).json() + pypi_hermes_version: str = pypi_hermes_json["info"]["version"] + + def version_tuple(version_string: str) -> tuple: + version_string = re.split(r"[A-Za-z]", version_string, maxsplit=1)[0] + return tuple(int(p) for p in version_string.split(".") if p) + + if version_tuple(current_hermes_version) < version_tuple(pypi_hermes_version): + sc.echo(f"You are using an old version of HERMES. ({current_hermes_version})", sc.Formats.WARNING) + sc.echo( + f"Please upgrade to the latest version ({pypi_hermes_version}) before running 'hermes init' to " + f"avoid errors!", + sc.Formats.FAIL) + elif version_tuple(current_hermes_version) == version_tuple(pypi_hermes_version): + sc.echo(f"Your version of HERMES ({current_hermes_version}) is up to date.", sc.Formats.OKGREEN) + else: + sc.echo( + f"Your version of HERMES ({current_hermes_version}) is even newer than " + f"the latest version ({pypi_hermes_version}).", sc.Formats.OKCYAN + sc.Formats.BOLD) + except Exception as e: + sc.echo(f"Could not fetch Pypi Hermes version. ({e})", sc.Formats.WARNING) + def test_initialization(self) -> None: """Test if init is possible and wanted. If not: sys.exit()""" sc.echo("Preparing HERMES initialization\n") @@ -307,7 +387,8 @@ def test_initialization(self) -> None: self.no_git_setup() sys.exit() - # Look at git remotes + # Look at git branch & remotes + self.git_branch = git_info.get_current_branch() remotes = git_info.get_remotes() if remotes: self.git_remote = remotes[0] @@ -415,7 +496,7 @@ def create_ci_template(self) -> None: """Downloads and configures the ci workflow files using templates from the chosen template branch.""" match self.git_hoster: case GitHoster.GitHub: - template_url = self.get_template_url("TEMPLATE_hermes_github_to_zenodo.yml") + template_url = self.get_template_url("hermes_github.yml") ci_file_folder = Path(".github/workflows") ci_file_name = "hermes_github.yml" ci_file_path = ci_file_folder / ci_file_name @@ -425,11 +506,12 @@ def create_ci_template(self) -> None: self.mark_as_new_path(ci_file_path) # Creating folder & ci file ci_file_folder.mkdir(parents=True, exist_ok=True) + sc.debug_info(f"Downloading github template from {template_url}") download_file_from_url(template_url, ci_file_path) self.configure_ci_template(ci_file_path) sc.echo(f"GitHub CI: File was created at {ci_file_path}", formatting=sc.Formats.OKGREEN) case GitHoster.GitLab: - gitlab_ci_template_url = self.get_template_url("TEMPLATE_hermes_gitlab_to_zenodo.yml") + gitlab_ci_template_url = self.get_template_url("hermes_gitlab.yml") hermes_ci_template_url = self.get_template_url("hermes-ci.yml") gitlab_ci_path = Path(".gitlab-ci.yml") gitlab_folder_path = Path("gitlab") @@ -452,6 +534,7 @@ def create_ci_template(self) -> None: sc.echo(f"GitLab CI: {gitlab_ci_path} was created.", formatting=sc.Formats.OKGREEN) self.configure_ci_template(gitlab_ci_path) # Creating hermes-ci + sc.debug_info(f"Downloading gitlab hermes-ci template from {hermes_ci_template_url}") download_file_from_url(hermes_ci_template_url, hermes_ci_path) self.configure_ci_template(hermes_ci_path) @@ -467,27 +550,26 @@ def create_ci_template(self) -> None: def configure_ci_template(self, ci_file_path) -> None: """Replaces all {%parameter%} in a ci file with values from ci_parameters dict""" - with open(ci_file_path, 'r') as file: - content = file.read() - parameters = list(set(re.findall(r'{%(.*?)%}', content))) - for parameter in parameters: - if parameter in self.ci_parameters: - value = str(self.ci_parameters[parameter]) - content = content.replace(f'{{%{parameter}%}}', value) - else: - sc.debug_info(f"CI File Parameter {{%{parameter}%}} was not set.", formatting=sc.Formats.WARNING) - content = content.replace(f'{{%{parameter}%}}', '') + jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(""), + block_start_string="{%%", block_end_string="%%}", + variable_start_string="{%", variable_end_string="%}") + source_text = Path(ci_file_path).read_text(encoding="utf-8") + used_params = jinja2.meta.find_undeclared_variables(jinja_env.parse(source_text)) + template = jinja_env.get_template(str(ci_file_path)) + missing = [p for p in used_params if p not in self.ci_parameters] + if missing: + sc.echo(f"CI Template has missing parameters: {missing}", formatting=sc.Formats.WARNING) + rendered = template.render(self.ci_parameters) with open(ci_file_path, 'w') as file: - file.write(content) + file.write(rendered) def create_zenodo_token(self) -> None: - """Makes the user create a zenodo token and saves it in self.tokens.""" - self.tokens[self.deposit_platform] = "" + """Makes the user create a zenodo token and saves it in self.deposit_platform.token.""" # Deactivated Zenodo OAuth as long as the refresh token bug is not fixed. if self.setup_method == "a": sc.echo("Doing OAuth with Zenodo is currently not available.") - if self.setup_method == "m" or self.tokens[self.deposit_platform] == '': - zenodo_token_url = urljoin(DepositPlatformUrls[self.deposit_platform], + if self.setup_method == "m" or self.deposit_platform.token == '': + zenodo_token_url = urljoin(self.deposit_platform.url, "account/settings/applications/tokens/new/") sc.echo("{} and create an access token.".format( sc.create_console_hyperlink(zenodo_token_url, "Open this link") @@ -497,18 +579,30 @@ def create_zenodo_token(self) -> None: sc.press_enter_to_continue() else: while True: - self.tokens[self.deposit_platform] = sc.answer("Enter the token here: ") - valid = connect_zenodo.test_if_token_is_valid(self.tokens[self.deposit_platform]) + self.deposit_platform.token = sc.answer("Enter the token here: ") + valid = connect_zenodo.test_if_token_is_valid(self.deposit_platform.token) if valid: - sc.echo(f"The token was validated by {connect_zenodo.name}.", + sc.echo(f"The token was validated by {self.deposit_platform.name}.", formatting=sc.Formats.OKGREEN) break else: - sc.echo(f"The token could not be validated by {connect_zenodo.name}. " + sc.echo(f"The token could not be validated by {self.deposit_platform.name}. " "Make sure to enter the complete token.\n" - "(If this error persists, you should try switching to the manual setup mode.)", + "(If this error persists, you should restart and switch to the manual setup mode.)", formatting=sc.Formats.WARNING) + def create_rodare_token(self): + token_url = urljoin(self.deposit_platform.url, "account/settings/applications/tokens/new/") + sc.echo("{} and create an access token.".format( + sc.create_console_hyperlink(token_url, "Open this link") + )) + sc.echo("It needs the scopes \"deposit:actions\" and \"deposit:write\".") + if self.setup_method == "m": + sc.press_enter_to_continue() + else: + # TODO try to validate the token + self.deposit_platform.token = sc.answer("Enter the token here: ") + def configure_git_project(self) -> None: """Adds the token to the git secrets & changes action workflow settings.""" match self.git_hoster: @@ -524,8 +618,8 @@ def configure_github(self) -> None: if self.tokens[GitHoster.GitHub]: sc.echo("OAuth at GitHub was successful.", formatting=sc.Formats.OKGREEN) sc.debug_info(github_token=self.tokens[GitHoster.GitHub]) - connect_github.create_secret(self.git_remote_url, "ZENODO_SANDBOX", - secret_value=self.tokens[self.deposit_platform], + connect_github.create_secret(self.git_remote_url, self.deposit_platform.token_name, + secret_value=self.deposit_platform.token, token=self.tokens[GitHoster.GitHub]) connect_github.allow_actions(self.git_remote_url, token=self.tokens[GitHoster.GitHub]) @@ -534,13 +628,14 @@ def configure_github(self) -> None: sc.echo("Something went wrong while doing OAuth. You'll have to do it manually instead.", formatting=sc.Formats.WARNING) if not oauth_success: - sc.echo("Add the {} token{} to your {} under the name ZENODO_SANDBOX.".format( + sc.echo("Add the {} token{} to your {} under the name {}.".format( self.deposit_platform.name, - f" ({self.tokens[self.deposit_platform]})" if self.tokens[self.deposit_platform] else "", + f" ({self.deposit_platform.token})" if self.deposit_platform.token else "", sc.create_console_hyperlink( self.git_remote_url + "/settings/secrets/actions", "project's GitHub secrets" - ) + ), + self.deposit_platform.token_name )) sc.press_enter_to_continue() sc.echo("Next open your {} and check the checkbox which reads:".format( @@ -569,7 +664,7 @@ def configure_gitlab(self) -> None: token = sc.answer("Then paste the token here: ") if gl.authorize(token): vars_created = gl.create_variable( - "ZENODO_TOKEN", self.tokens[self.deposit_platform], + self.deposit_platform.token_name, self.deposit_platform.token, f"This token is used by Hermes to publish on {self.deposit_platform.name}." ) if vars_created: @@ -601,30 +696,56 @@ def configure_gitlab(self) -> None: sc.echo("Then, add that token as variable with key HERMES_PUSH_TOKEN.") sc.echo("(For your safety, you should set the visibility to 'Masked and hidden'.)") sc.press_enter_to_continue() - sc.echo("Next, add the {} token{} as variable with key ZENODO_TOKEN.".format( + sc.echo("Next, add the {} token{} as variable with key {}.".format( self.deposit_platform.name, - f" ({self.tokens[self.deposit_platform]})" if self.tokens[self.deposit_platform] else "" + f" ({self.deposit_platform.token})" if self.deposit_platform.token else "", + self.deposit_platform.token_name )) sc.echo("(For your safety, you should set the visibility to 'Masked and hidden'.)") sc.press_enter_to_continue() def choose_deposit_platform(self) -> None: """User chooses his desired deposit platform.""" - deposit_platform_list = list(DepositPlatformNames.keys()) deposit_platform_index = sc.choose( - "Where do you want to publish the software?", [DepositPlatformNames[dp] for dp in deposit_platform_list] + "Where do you want to publish the software?", [do.name for do in DepositOptions] ) - self.deposit_platform = deposit_platform_list[deposit_platform_index] + self.deposit_platform = DepositOptions[deposit_platform_index] def integrate_deposit_platform(self) -> None: """Makes changes to the toml data or something else based on the chosen deposit platform.""" - deposit_url = DepositPlatformUrls.get(self.deposit_platform) - self.hermes_toml_data["deposit"]["invenio_rdm"]["site_url"] = deposit_url + deposit_plugin: str = self.deposit_platform.plugin_name + self.hermes_toml_data["deposit"]["target"] = deposit_plugin + self.hermes_toml_data["deposit"][deposit_plugin] = {} + self.hermes_toml_data["deposit"][deposit_plugin]["site_url"] = self.deposit_platform.url + self.ci_parameters["deposit_parameter_token"] = f"-O {deposit_plugin}.auth_token" + self.ci_parameters["deposit_token_name"] = self.deposit_platform.token_name + + if deposit_plugin.startswith("invenio") or deposit_plugin.startswith("rodare"): + # Invenio & rodare need access_right + # For possible customization we ask the user here + options = ["open", "closed", "restricted", "embargoed"] + target_access_right_index = sc.choose( + text="Select an access right for your publication", + options=options + ) + target_access_right = options[target_access_right_index] + self.hermes_toml_data["deposit"][deposit_plugin]["access_right"] = target_access_right + if target_access_right == "restricted": + conditions = sc.answer("Enter the access conditions of the restriction: ") + self.hermes_toml_data["deposit"][deposit_plugin]["access_conditions"] = conditions + elif target_access_right == "embargoed": + embargo_date = sc.answer("Enter the embargo date (YYYY-MM-DD): ") + self.hermes_toml_data["deposit"][deposit_plugin]["embargo_date"] = embargo_date + + if deposit_plugin.startswith("rodare"): + # Rodare needs the robis_pub_id + robis_pub_id = sc.answer("Enter the corresponding Robis Publication ID: ") + self.hermes_toml_data["deposit"][deposit_plugin]["robis_pub_id"] = robis_pub_id def choose_setup_method(self) -> None: """User chooses his desired setup method: Either preferring automatic (if available) or manual.""" setup_method_index = sc.choose( - f"How do you want to connect {DepositPlatformNames[self.deposit_platform]} " + f"How do you want to connect {self.deposit_platform.name} " f"with your {self.git_hoster.name} CI?", options=[ "Automatically (using OAuth / Device Flow)", @@ -635,14 +756,18 @@ def choose_setup_method(self) -> None: def connect_deposit_platform(self) -> None: """Acquires the access token of the chosen deposit platform.""" - assert self.deposit_platform != DepositPlatform.Empty - match self.deposit_platform: - case DepositPlatform.Zenodo: - connect_zenodo.setup(using_sandbox=False) - self.create_zenodo_token() - case DepositPlatform.ZenodoSandbox: - connect_zenodo.setup(using_sandbox=True) - self.create_zenodo_token() + used_deposit_plugin = self.deposit_platform.plugin_name + deposit_url = self.deposit_platform + deposit_name = self.deposit_platform.name + if used_deposit_plugin.startswith("invenio"): + connect_zenodo.setup(zenodo_url=self.deposit_platform.url, display_name=self.deposit_platform.name) + self.create_zenodo_token() + elif used_deposit_plugin.startswith("rodare"): + self.create_rodare_token() + else: + sc.echo(f"Unknown deposit plugin: {used_deposit_plugin}", formatting=sc.Formats.WARNING) + sc.echo(f"Getting an access token from {deposit_name} ({deposit_url}) is not supported by hermes init." + "You might have to do it manually instead.", formatting=sc.Formats.WARNING) def choose_plugins(self) -> None: """User chooses the plugins he wants to use.""" @@ -718,35 +843,90 @@ def no_git_setup(self, start_question: str = "") -> None: sc.echo("\nHERMES is now initialized (without git integration or CI/CD files).\n", formatting=sc.Formats.OKGREEN) - def choose_push_branch(self) -> None: - """User chooses the branch that should be used to activate the whole hermes process.""" + def choose_push_trigger(self) -> None: + """User chooses the branch / tag that should be used to trigger the whole hermes pipeline.""" push_choice = sc.choose( "When should the automated HERMES process start?", [ - "When I push a branch", - "When I push a specific tag (not implemented)", + "When I push on target branch", + f"When I push on current branch ({self.git_branch})", + "When I push any tag", + "When I push a tag with target pattern" ] ) if push_choice == 0: - branch = sc.answer("Enter target branch: ") - self.ci_parameters["push_branch"] = branch - sc.echo(f"The HERMES pipeline will be activated when you push on {sc.Formats.BOLD.wrap_around(branch)}", - formatting=sc.Formats.OKGREEN) - sc.echo() + branch_suggestion_max_count = 9 + branch_suggestions: list[str] = git_info.run_git_command( + "for-each-ref --sort=-committerdate refs/heads/ --format='%(refname:short)'" + ).split("\n") + branch_suggestions = [b.strip() for b in branch_suggestions if b.strip() != ""] + branch_suggestions.sort() + branch_suggestions.sort(key=lambda branch_name: len(branch_name)) + branch_suggestions = branch_suggestions[:branch_suggestion_max_count] + branch_count = len(branch_suggestions) + branch_suggestions.append("Custom branch name") + branch_choice = sc.choose("Choose target branch: ", branch_suggestions) + if branch_choice < branch_count: + self.set_push_trigger_to_branch(branch_suggestions[branch_choice].removeprefix("'").removesuffix("'")) + else: + branch = sc.answer("Enter custom branch name: ") + self.set_push_trigger_to_branch(branch) elif push_choice == 1: - sc.echo("Setting up triggering by tags is currently not implemented.", formatting=sc.Formats.WARNING) - sc.echo(f"You can visit {TUTORIAL_URL} to set it up manually later-on.", formatting=sc.Formats.WARNING) + self.set_push_trigger_to_branch(self.git_branch) + elif push_choice == 2: + self.set_push_trigger_to_tag() + elif push_choice == 3: + pattern_hint = "" + if self.git_hoster == GitHoster.GitHub: + pattern_hint = " (GitHub uses glob patterns)" + elif self.git_hoster == GitHoster.GitLab: + pattern_hint = " (Gitlab uses regex)" + pattern = sc.answer(f"Enter the target tag-pattern{pattern_hint}: ") + self.set_push_trigger_to_tag(pattern) + + def set_push_trigger_to_branch(self, branch: str) -> None: + """Sets the CI parameters, so that the pipeline gets triggered when the branch gets pushed.""" + self.ci_parameters["gh_push_branches_or_tags"] = "branches" + self.ci_parameters["gh_push_target"] = branch + self.ci_parameters["gl_push_condition"] = f"$CI_COMMIT_BRANCH == \"{branch}\"" + bold_branch = sc.Formats.BOLD.wrap_around(branch) + sc.echo(f"The HERMES pipeline will be activated when you push on {bold_branch}.", + formatting=sc.Formats.OKGREEN) + sc.echo() + + def set_push_trigger_to_tag(self, tag_pattern: str = "") -> None: + """ + Sets the CI parameters, so that the pipeline gets triggered when a tag that matches the pattern gets pushed. + """ + self.ci_parameters["gh_push_branches_or_tags"] = "tags" + self.ci_parameters["gh_create_curate_branch"] = 'git checkout -b "hermes/curate-$SHORT_SHA" ${{ github.ref }}' + self.ci_parameters["gl_create_curate_branch"] = 'git checkout -b "$MR_TARGET_BRANCH" "$CI_COMMIT_REF_NAME"' + if tag_pattern: + self.ci_parameters["gh_push_target"] = f"\"{tag_pattern}\"" + self.ci_parameters["gl_push_condition"] = f"$CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /{tag_pattern}/" + bold_pattern = sc.Formats.BOLD.wrap_around(tag_pattern) + sc.echo(f"The HERMES pipeline will be activated when you push a tag that fits '{bold_pattern}'.", + formatting=sc.Formats.OKGREEN) + else: + self.ci_parameters["gh_push_target"] = "\"*\"" + self.ci_parameters["gl_push_condition"] = "$CI_COMMIT_TAG" + sc.echo("The HERMES pipeline will be activated when you push a tag.", formatting=sc.Formats.OKGREEN) + sc.echo() def choose_deposit_files(self) -> None: """User chooses the files that should be included in the deposition.""" - dp_name = DepositPlatformNames[self.deposit_platform] + dp_name = self.deposit_platform.name add_readme = False if self.folder_info.has_readme: if sc.confirm(f"Do you want to append your README.md to the {dp_name} upload?"): self.ci_parameters["deposit_extra_files"] = "--file README.md " add_readme = True + else: + self.ci_parameters["deposit_extra_files"] = "" + add_readme = False options = [ - "All (non hidden) folders", + "Nothing else", + "All (visible) folders", "Everything (all folders & all files)", "Enter a custom list of paths", ] @@ -757,37 +937,53 @@ def choose_deposit_files(self) -> None: file_choice = sc.choose(f"Which{_other} folders / files of your root directory " f"should be included in the {dp_name} upload?", options=options) match file_choice: - case 0: + case 0: # Nothing + self.ci_parameters["deposit_zip_files"] = "-" + case 1: # All folders self.ci_parameters["deposit_zip_files"] = " ".join(self.folder_info.dir_folders) - case 1: + case 2: # All folders all files self.ci_parameters["deposit_zip_files"] = "" - case 2: + case 3: # Custom List custom_files = [] while True: custom_path = sc.answer("Enter a path you want to include (enter nothing if you are done): ") - if custom_path == "": + if custom_path.strip() == "": break if os.path.exists(os.path.join(self.folder_info.current_dir, custom_path)): custom_files.append(custom_path) sc.echo(f"{custom_path} has been added.", formatting=sc.Formats.OKGREEN) else: sc.echo(f"{custom_path} does not exist.", formatting=sc.Formats.FAIL) - self.ci_parameters["deposit_zip_files"] = " ".join(custom_files) + if custom_files: + self.ci_parameters["deposit_zip_files"] = " ".join(custom_files) + else: + self.ci_parameters["deposit_zip_files"] = "-" case _: index = int(file_choice) - folder_base_index if 0 <= index < len(self.folder_info.dir_folders): self.ci_parameters["deposit_zip_files"] = self.folder_info.dir_folders[index] + self.ci_parameters["deposit_parameter_zip_file"] = "--file " + self.ci_parameters["deposit_zip_name"] + if self.ci_parameters["deposit_zip_files"] == "-": + self.ci_parameters["deposit_parameter_zip_file"] = "" + # Print selection to confirm sc.echo("Your upload will consist of the following:") if add_readme: sc.echo("\tUnzipped:", formatting=sc.Formats.BOLD) sc.echo("\t\tREADME.md", formatting=sc.Formats.OKCYAN) sc.echo("\tZipped:", formatting=sc.Formats.BOLD) - if self.ci_parameters["deposit_zip_files"] != "": - for file in self.ci_parameters["deposit_zip_files"].split(" "): + if self.ci_parameters["deposit_zip_files"] == "-": + sc.echo("\t\t-", formatting=sc.Formats.OKCYAN) + elif self.ci_parameters["deposit_zip_files"] == "": + for file in self.folder_info.dir_list: sc.echo(f"\t\t{file}", formatting=sc.Formats.OKCYAN) else: - for file in self.folder_info.dir_list: + for file in self.ci_parameters["deposit_zip_files"].split(" "): sc.echo(f"\t\t{file}", formatting=sc.Formats.OKCYAN) + if not sc.confirm("Do you want to confirm your selection?"): + sc.echo("Your selection was cleared. Now you can select the files again.") + self.choose_deposit_files() + else: + sc.echo("You can change the selected files later inside the CI file or by running 'hermes init' again.") def mark_as_new_path(self, path: Path, avoid_existing: bool = True) -> None: """ diff --git a/src/hermes/commands/init/util/connect_github.py b/src/hermes/commands/init/util/connect_github.py index e2865dd2..a12bd693 100644 --- a/src/hermes/commands/init/util/connect_github.py +++ b/src/hermes/commands/init/util/connect_github.py @@ -2,14 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileContributor: Nitai Heeb -import requests from base64 import b64encode + +import requests from nacl import encoding, public from . import slim_click as sc from .oauth_process import OauthProcess - local_port = 8333 client_id = 'Ov23linvdC7WzHnOO2WK' client_secret = 'empty-as-not-needed-for-public-device-flow' diff --git a/src/hermes/commands/init/util/connect_gitlab.py b/src/hermes/commands/init/util/connect_gitlab.py index 93cbfbdd..9bcfe629 100644 --- a/src/hermes/commands/init/util/connect_gitlab.py +++ b/src/hermes/commands/init/util/connect_gitlab.py @@ -2,14 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileContributor: Nitai Heeb -import requests -from urllib.parse import urlparse, urljoin, quote from datetime import datetime, timedelta +from urllib.parse import quote, urljoin, urlparse + +import requests from . import slim_click as sc from .oauth_process import OauthProcess - default_scopes = "api write_repository" device_code_addition = "oauth/authorize_device" token_addition = "oauth/token" diff --git a/src/hermes/commands/init/util/connect_zenodo.py b/src/hermes/commands/init/util/connect_zenodo.py index e9873cb5..530533c5 100644 --- a/src/hermes/commands/init/util/connect_zenodo.py +++ b/src/hermes/commands/init/util/connect_zenodo.py @@ -3,8 +3,9 @@ # SPDX-FileContributor: Nitai Heeb import time -import requests + import oauthlib.oauth2.rfc6749.errors +import requests from . import slim_click as sc from .oauth_process import OauthProcess @@ -20,10 +21,8 @@ local_port = 8334 sandbox_client_id = 'QJ8Q9GBI78uOdNmVNK1Vd0oAOJHqmYGvxRxiSFxt' sandbox_client_secret = 'nGuOqoDtd2tckP6lmQS3If3cY39lPLKLU8skcv72JeowNupMD2bnLparsGO9' -sandbox_base_url = 'https://sandbox.zenodo.org' real_client_id = 'L0d9HQVW4Ig9PnC6qh6zkOAwgvYy08GcmHJqVVvV' real_client_secret = '0HIvtC2D2aPvpq2W0GtfWdeivwkqvnvrOTGx14nUJA5lDXrEDSaQAnqxHbLH' -real_base_url = 'https://zenodo.org' url_suffix_authorize = '/oauth/authorize' url_suffix_token = '/oauth/token' url_suffix_api_list = '/api/deposit/depositions' @@ -32,7 +31,7 @@ client_id = client_secret = base_url = authorize_url = token_url = api_list_url = name = "" -def setup(using_sandbox: bool = USING_SANDBOX_AS_DEFAULT): +def setup(zenodo_url="https://sandbox.zenodo.org/", display_name=""): global client_id global client_secret global base_url @@ -40,13 +39,16 @@ def setup(using_sandbox: bool = USING_SANDBOX_AS_DEFAULT): global token_url global api_list_url global name - client_id = sandbox_client_id if using_sandbox else real_client_id - client_secret = sandbox_client_secret if using_sandbox else real_client_secret - base_url = sandbox_base_url if using_sandbox else real_base_url + base_url = zenodo_url authorize_url = base_url + url_suffix_authorize token_url = base_url + url_suffix_token api_list_url = base_url + url_suffix_api_list - name = "Zenodo (Sandbox)" if using_sandbox else "Zenodo" + using_sandbox = "sandbox" in zenodo_url + client_id = sandbox_client_id if using_sandbox else real_client_id + client_secret = sandbox_client_secret if using_sandbox else real_client_secret + name = display_name + if name == "": + name = "Zenodo Sandbox" if using_sandbox else "Zenodo" setup() @@ -79,7 +81,7 @@ def test_if_token_is_valid(token: str) -> bool: def test_if_refresh_token_authorization_works(): - for version in [True, False]: + for version in ["https://sandbox.zenodo.org/", "https://zenodo.org/"]: setup(version) sc.echo(f"Testing if the {name} refresh token mechanism works...", formatting=sc.Formats.BOLD+sc.Formats.WARNING) diff --git a/src/hermes/commands/init/util/git_info.py b/src/hermes/commands/init/util/git_info.py index e26dcf92..f1db3077 100644 --- a/src/hermes/commands/init/util/git_info.py +++ b/src/hermes/commands/init/util/git_info.py @@ -2,12 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileContributor: Nitai Heeb -import re import os +import re import subprocess from pathlib import Path - default_cwd = "" @@ -31,7 +30,7 @@ def get_valid_cwd(cwd="") -> str: return str(path) -def run_git_command(command: str, cwd="") -> str: +def run_git_command(command: str, cwd="", throw_exception=False) -> str: """ Runs any git command using subprocess. Raises Exception when returncode != 0. :param command: The command as string with or without the 'git' main command. @@ -45,7 +44,7 @@ def run_git_command(command: str, cwd="") -> str: if command_list[0] != "git": command_list.insert(0, "git") # Run subprocess - result = subprocess.run(command_list, cwd=cwd, capture_output=True, text=True) + result = subprocess.run(command_list, cwd=cwd, capture_output=True, text=True, check=throw_exception) # Return output or error if result.returncode != 0: raise Exception(result.stderr) @@ -88,11 +87,11 @@ def get_current_branch() -> str: """ Returns the name of the current branch. """ - branch_info = run_git_command("branch") - for line in branch_info.splitlines(): - if line.startswith("*"): - return line.split()[1].strip() - raise Exception("Current branch not found.") + try: + branch_name = run_git_command("rev-parse --abbrev-ref HEAD", throw_exception=True) + return branch_name.strip() + except subprocess.CalledProcessError: + return "" def is_git_installed() -> bool: diff --git a/src/hermes/commands/init/util/oauth_process.py b/src/hermes/commands/init/util/oauth_process.py index 6ef210a6..33439f74 100644 --- a/src/hermes/commands/init/util/oauth_process.py +++ b/src/hermes/commands/init/util/oauth_process.py @@ -4,19 +4,20 @@ # SPDX-FileContributor: David Pape from __future__ import annotations -import logging +import json +import logging import os import threading import time import webbrowser -import requests -import requests_oauthlib -import json -from threading import Event from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Event from urllib.parse import parse_qs, urlparse +import requests +import requests_oauthlib + from . import slim_click as sc PREFER_DEVICE_FLOW = True diff --git a/src/hermes/utils.py b/src/hermes/utils.py index 56c4590c..d6464321 100644 --- a/src/hermes/utils.py +++ b/src/hermes/utils.py @@ -40,7 +40,8 @@ def retrieve_project_urls(metadata_urls: list[str]) -> dict[str, str]: # Publication metadata # TODO: Fetch this from somewhere -hermes_doi = "10.5281/zenodo.13311079" # hermes v0.8.1 +hermes_doi = "10.5281/zenodo.14931650" # "10.5281/zenodo.13311079" for hermes v0.8.1 +hermes_doi_url = "https://doi.org/10.5281/zenodo.14931650" hermes_concept_doi = "10.5281/zenodo.13221383" """Fix "concept" DOI that always refers to the newest version."""