diff --git a/TEMPLATE_hermes_github_to_zenodo.yml b/TEMPLATE_hermes_github_to_zenodo.yml index 3f6b8ab..c7dd404 100644 --- a/TEMPLATE_hermes_github_to_zenodo.yml +++ b/TEMPLATE_hermes_github_to_zenodo.yml @@ -48,7 +48,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install hermes + - run: pip install git+https://github.com/softwarepub/hermes.git - run: hermes harvest - run: hermes process - run: hermes curate @@ -92,12 +92,12 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install hermes + - run: pip install git+https://github.com/softwarepub/hermes.git # ADAPT # If you want to publish artifacts (e.g., a zipped snapshot of your repository), # you can prepare this here. - - run: git archive --format zip HEAD src > showcase.zip + - run: git archive --format zip HEAD src > artifact.zip # Run the HERMES deposition and postprocessing steps. # ADAPT @@ -106,7 +106,7 @@ jobs: # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), # rather than a new version of the same collection! - - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file showcase.zip --file README.md + - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file artifact.zip --file README.md # ADAPT # Remove this command if you don't want to do any postprocessing diff --git a/gitlab/hermes-ci.yml b/gitlab/hermes-ci.yml index 849dc88..c163ccf 100644 --- a/gitlab/hermes-ci.yml +++ b/gitlab/hermes-ci.yml @@ -74,7 +74,7 @@ python -m venv .hermes-env fi - . .hermes-env/bin/activate - - pip install hermes + - pip install git+https://github.com/softwarepub/hermes.git tags: - docker artifacts: diff --git a/init-templates/hermes-ci.yml b/init-templates/hermes-ci.yml new file mode 100644 index 0000000..3e6c38a --- /dev/null +++ b/init-templates/hermes-ci.yml @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf +# +# SPDX-License-Identifier: CC0-1.0 + +# "Call-back" to create archive +.hermes_create_deposition: &hermes_create_deposition + - git fetch origin "$MR_TARGET_BRANCH" + - git archive --format zip "origin/$MR_TARGET_BRANCH" {%deposit_zip_files%} > $HERMES_ARCHIVE + +# Basic building blocks of hermes jobs +# - Set up git with a valid author and configure URL to allow pushes. +.hermes_prepare_git: &hermes_prepare_git + - git config user.name "$GITLAB_USER_NAME" + - git config user.email "$GITLAB_USER_EMAIL" + - git remote set-url origin "${CI_SERVER_PROTOCOL}://__token__:${HERMES_PUSH_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/${CI_PROJECT_PATH}.git" + +# - Switch to new MR_SOURCE_BRANCH. +.hermes_switch_branch: &hermes_switch_branch + - git branch "$MR_SOURCE_BRANCH" "origin/$MR_TARGET_BRANCH" + - git switch "$MR_SOURCE_BRANCH" + +# - Commit changes and create a matching merge request. +.hermes_create_merge_request: &hermes_create_merge_request + - | + if git diff --cached --quiet; then + echo "No staged changes. Skipping MR creation." + else + git commit -m "$MR_COMMIT_MESSAGE" + git push origin "$MR_SOURCE_BRANCH" \ + -o merge_request.create \ + -o merge_request.target="$MR_TARGET_BRANCH" \ + -o merge_request.title="$MR_TITLE" \ + -o merge_request.description="$MR_DESCRIPTION" \ + -o merge_request.remove_source_branch + fi + +# - Delete all branches created by hermes for curation. +.hermes_cleanup_branches: &hermes_cleanup_branches + - | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done + +# Base job template for hermes +.hermes_job: + stage: deploy + image: python:3.10 + cache: + key: hermes + paths: + - .hermes-env + before_script: + - *hermes_prepare_git + - | + if test ! -d .hermes-env + then + python -m venv .hermes-env + fi + - . .hermes-env/bin/activate + - {%pip_install_hermes%} +{%pip_install_plugins_gitlab%} + tags: + - docker + artifacts: + paths: [".hermes/", "hermes.log"] + when: always + +# hermes metadata preparation - this job creates a MR that will trigger deposition when merged. +.hermes_curate: + extends: + - .hermes_job + variables: + MR_BASE_REF: "origin/$CI_COMMIT_BRANCH" + MR_SOURCE_BRANCH: hermes/curate-result-$CI_COMMIT_SHORT_SHA + MR_TARGET_BRANCH: hermes/curate-$CI_COMMIT_SHORT_SHA + MR_COMMIT_MESSAGE: "[hermes] Add metadata for curation" + MR_TITLE: Metadata Curation for Commit $CI_COMMIT_SHORT_SHA + MR_DESCRIPTION: >- + Please carefully review the attached metadata. + If you are satisfied with the result, you may merge this MR, which will trigger publication. + (Any temporary branches will be cleaned up.) + rules: + - exists: + - .hermes/curate/target_branch + when: never + - if: $CI_COMMIT_TITLE =~ /^Merge branch 'hermes\/post-[0-9a-f]{8}' into/ + when: never + - if: $CI_PIPELINE_SOURCE == "push" && {%gl_push_condition%} + - when: never + script: + # Run hermes pipeline + - hermes harvest + - hermes process + - hermes curate + + # Store branch & commit ref for post-processing + - | + if [ -z "$MR_POST_BRANCH" ] + then + if [ ! -z "$CI_COMMIT_BRANCH" ] + then + MR_POST_BRANCH=$CI_COMMIT_BRANCH + else + MR_POST_BRANCH="" + fi + fi + echo $MR_POST_BRANCH > .hermes/curate/target_branch + echo $CI_COMMIT_REF_NAME > .hermes/curate/commit_ref + + # Create target branch for curation merge + - {%gl_create_curate_branch|default('git branch "$MR_TARGET_BRANCH" "$MR_BASE_REF"', true)%} + - git push origin "$MR_TARGET_BRANCH" + + # Create curation branch and merge request + - *hermes_switch_branch + - git add -f .hermes/curate + - *hermes_create_merge_request + +# hermes deposition and post-processing - this job creates a MR to re-integrate the results of post-processing. +.hermes_deposit: + stage: deploy + extends: + - .hermes_job + variables: + HERMES_ARCHIVE: {%deposit_zip_name%} + MR_SOURCE_BRANCH: hermes/post-$CI_COMMIT_SHORT_SHA + MR_COMMIT_MESSAGE: "[hermes] Add post-processing results" + MR_TITLE: Review hermes post-processing results + MR_DESCRIPTION: >- + This is an automated pull request created by HERMES post-processing. + Please carefully review the changes and finally merge them into your + rules: + - if: $CI_PIPELINE_SOURCE != "push" || $CI_COMMIT_BRANCH !~ /^hermes\/curate-[0-9a-f]{8}/ + when: never + - exists: + - .hermes/curate/target_branch + script: + # Restore target branch + - MR_TARGET_BRANCH="$(cat .hermes/curate/target_branch)" + + # Get target branch from tag ref if empty + - | + # Test if empty and .hermes/curate/commit_ref exists + if [ -z "$MR_TARGET_BRANCH" ] && [ -s .hermes/curate/commit_ref ]; then + # Get default branch + DEFAULT_BRANCH="$CI_DEFAULT_BRANCH" + + # Take tag ref from .hermes/curate/commit_ref + REF="$(tr -d '\r\n' < .hermes/curate/commit_ref)" + TAG="${REF#refs/tags/}" + + # Get commit + git fetch --force --tags origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 || true + TAG_COMMIT="$(git rev-parse "${TAG}^{commit}" 2>/dev/null)" || exit 0 + + # Check if the default or another branch contains the tag commit + if git merge-base --is-ancestor "$TAG_COMMIT" "origin/$DEFAULT_BRANCH"; then + MR_TARGET_BRANCH="$DEFAULT_BRANCH" + else + BRANCH_REF="$( + git for-each-ref --contains "$TAG_COMMIT" --format='%(refname:short)' refs/remotes/origin \ + | grep -vE '^origin/(hermes/|HEAD$)' \ + | head -n1 + )" + if [ -n "$BRANCH_REF" ]; then + MR_TARGET_BRANCH="${BRANCH_REF#refs/heads/}" + fi + fi + fi + + {%% if deposit_parameter_zip_file %%} + # Invoke callback to create {%deposit_zip_name%} and publish + - *hermes_create_deposition + {%% endif %%} + # Call deposit to publish + - hermes deposit {%deposit_initial%} {%deposit_parameter_token%} "${%deposit_token_name%}" {%deposit_parameter_zip_file%} {%deposit_extra_files%} + + # Skip on missing target branch + - | + if [ -z "$MR_TARGET_BRANCH" ]; then + echo "MR_TARGET_BRANCH is empty. Skipping postprocess." + exit 0 + fi + + # Prepare clean branch for post-processing and run it + - *hermes_switch_branch + - hermes postprocess + + # Put all changes into a merge request (all but .hermes-env) + - git add . + - git reset .hermes-env + - *hermes_create_merge_request + after_script: + - *hermes_cleanup_branches + +.hermes_cleanup: + script: + - *hermes_cleanup_branches diff --git a/init-templates/hermes_github.yml b/init-templates/hermes_github.yml new file mode 100644 index 0000000..07e1d5b --- /dev/null +++ b/init-templates/hermes_github.yml @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf +# +# SPDX-License-Identifier: CC0-1.0 + + +name: Software Publication + +on: + # This trigger on {%gh_push_branches_or_tags%} being pushed starts the HERMES pipeline. + push: + {%gh_push_branches_or_tags%}: + - {%gh_push_target%} + + # This trigger on closed pull requests is mandatory for the rest of the HERMES workflow. + pull_request: + types: + - closed + +jobs: + hermes-prepare: + name: Prepare Metadata for Curation + runs-on: ubuntu-latest + + if: > + github.event_name == 'push' && ! ( + startsWith(github.ref_name, 'hermes/') || + contains(github.event.head_commit.message, 'hermes/post') + ) + + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: {%pip_install_hermes%} +{%pip_install_plugins_github%} + - run: hermes harvest + - run: hermes process + - run: hermes curate + + - run: | + # Cache current branch for PR close job + git branch --show-current > .hermes/curate/target_branch + echo "${{ github.ref }}" > .hermes/curate/commit_ref + + # Shorten the SHA for the PR title + SHORT_SHA=$(echo "$GITHUB_SHA" | cut -c -8) + echo "SHORT_SHA=$SHORT_SHA" >> "$GITHUB_ENV" + + # Create a curation branch + {%gh_create_curate_branch|default('git branch -c "hermes/curate-$SHORT_SHA"', true)%} + git push origin "hermes/curate-$SHORT_SHA" + + # Explicitly add to-be-curated metadata (which is ignored via .gitignore!) + git add -f .hermes/curate + - uses: peter-evans/create-pull-request@v7 + with: + base: hermes/curate-${{ env.SHORT_SHA }} + branch: hermes/curate-result-${{ env.SHORT_SHA }} + title: Metadata Curation for Commit ${{ env.SHORT_SHA }} + body: | + Please carefully review the attached metadata. + If you are satisfied with the result, you may merge this PR, which will trigger publication. + (Any temporary branches will be cleaned up.) + delete-branch: true + + hermes-curate: + name: Publish Software with Curated Metadata + if: github.event.pull_request.merged == true && startsWith( github.base_ref , 'hermes/curate-') + + runs-on: ubuntu-latest + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: {%pip_install_hermes%} +{%pip_install_plugins_github%} + {%% if deposit_parameter_zip_file %%} + # Archive the artifacts that will be published + - run: git archive --format zip HEAD {%deposit_zip_files%} > {%deposit_zip_name%} + + {%% endif %%} + # Run the HERMES deposition step + - run: hermes deposit {%deposit_initial%} {%deposit_parameter_token%} ${{ secrets.{%deposit_token_name%} }} {%deposit_parameter_zip_file%} {%deposit_extra_files%} + + # Run the HERMES postprocess step + - run: hermes postprocess + + # Prepare pull request to include the post-processing results + - run: echo "TARGET_BRANCH=$(cat .hermes/curate/target_branch)" >> "$GITHUB_ENV" + + # Take target_branch from tag ref if empty + - run: | + # Test if empty and .hermes/curate/commit_ref exists + if [ -z "$MR_TARGET_BRANCH" ] && [ -s .hermes/curate/commit_ref ]; then + # Get default branch + DEFAULT_BRANCH="${{ github.event.repository.default_branch }}" + + # Take tag ref from .hermes/curate/commit_ref + REF="$(tr -d '\r\n' < .hermes/curate/commit_ref)" + TAG="${REF#refs/tags/}" + + # Get commit + git fetch --force --tags origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 || true + TAG_COMMIT="$(git rev-parse "${TAG}^{commit}" 2>/dev/null)" || exit 0 + + # Check if the default or another branch contains the tag commit + if git merge-base --is-ancestor "$TAG_COMMIT" "origin/$DEFAULT_BRANCH"; then + echo "TARGET_BRANCH=$DEFAULT_BRANCH" >> "$GITHUB_ENV" + else + BRANCH_REF="$( + git for-each-ref --contains "$TAG_COMMIT" --format='%(refname:short)' refs/remotes/origin \ + | grep -vE '^origin/(hermes/|HEAD$)' \ + | head -n1 + )" + if [ -n "$BRANCH_REF" ]; then + echo "TARGET_BRANCH=${BRANCH_REF#refs/heads/}" >> "$GITHUB_ENV" + fi + fi + fi + + # Create the pull request + - uses: peter-evans/create-pull-request@v5 + if: ${{ env.TARGET_BRANCH != '' }} + with: + branch: hermes/post-${{ github.run_id }} + base: ${{ env.TARGET_BRANCH }} + title: Review hermes post-processing results + body: | + This is an automated pull request created by HERMES post-processing. + + Please carefully review the changes and finally merge them into your + + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done + + + hermes-cleanup: + name: Cleanup aborted curation branches + if: github.event.pull_request.merged == false && startsWith( github.base_ref , 'hermes/curate-') + + runs-on: ubuntu-latest + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v4 + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done diff --git a/init-templates/hermes_gitlab.yml b/init-templates/hermes_gitlab.yml new file mode 100644 index 0000000..05edccf --- /dev/null +++ b/init-templates/hermes_gitlab.yml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf +# +# SPDX-License-Identifier: CC0-1.0 + +include: gitlab/hermes-ci.yml + +# "Call-back" to create archive +.hermes_create_deposition: &hermes_create_deposition + - git fetch origin "$MR_TARGET_BRANCH" + - git archive --format zip "origin/$MR_TARGET_BRANCH" > $HERMES_ARCHIVE_NAME + +hermes_curate: + extends: + - .hermes_curate + +hermes_deposit: + extends: + - .hermes_deposit