From 8aa20fa16776dd7dd90c1ab03f0425d2a979e271 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Sat, 13 Dec 2025 18:46:27 +0100 Subject: [PATCH 1/7] chore: Utilize Apify's wait-for-checks --- .github/workflows/pre_release.yaml | 54 +++++------------------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml index 40def076..91b43492 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/pre_release.yaml @@ -32,54 +32,18 @@ jobs: release_type: prerelease existing_changelog_path: CHANGELOG.md - actions_lint_check: - name: Actions lint check - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Run actionlint - uses: rhysd/actionlint@v1.7.9 - - lint_check: - name: Lint check - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - - type_check: - name: Type check - uses: apify/workflows/.github/workflows/python_type_check.yaml@main - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - - unit_tests: - name: Unit tests - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - secrets: inherit - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - operating-systems: '["ubuntu-latest", "windows-latest"]' - python-version-for-codecov: "3.14" - operating-system-for-codecov: ubuntu-latest - - async_docstrings: - name: Async dostrings check - uses: ./.github/workflows/_async_docstrings_check.yaml - - integration_tests: - name: Integration tests - uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - secrets: inherit + # If github.ref points to a [ci skip] commit, this workflow won't run. + # Otherwise, these checks will be launched from the `run_code_checks` workflow. + wait_for_checks: + name: Wait for code checks to pass + uses: apify/workflows/.github/workflows/wait_for_checks.yaml@wait-for-checks-action with: - python-versions: '["3.10", "3.14"]' - operating-systems: '["ubuntu-latest"]' - python-version-for-codecov: "3.14" - operating-system-for-codecov: ubuntu-latest + ref: ${{ github.ref }} + check-regexp: "( check| tests)$" update_changelog: name: Update changelog - needs: [release_metadata, lint_check, type_check, unit_tests, integration_tests] + needs: [release_metadata, wait_for_checks] uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main with: version_number: ${{ needs.release_metadata.outputs.version_number }} @@ -89,7 +53,7 @@ jobs: publish_to_pypi: name: Publish to PyPI - needs: [release_metadata, update_changelog] + needs: [release_metadata, wait_for_checks, update_changelog] runs-on: ubuntu-latest permissions: contents: write From 899c95a275b5d53e86c451d13769052bd1cd7691 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Sun, 14 Dec 2025 17:56:57 +0100 Subject: [PATCH 2/7] chore(ci): Align GitHub workflows across Python projects --- .github/workflows/build_and_deploy_docs.yaml | 4 +++- .github/workflows/pre_release.yaml | 15 +++++---------- .github/workflows/run_code_checks.yaml | 3 +++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_deploy_docs.yaml b/.github/workflows/build_and_deploy_docs.yaml index 7fd422c9..e77a41a3 100644 --- a/.github/workflows/build_and_deploy_docs.yaml +++ b/.github/workflows/build_and_deploy_docs.yaml @@ -7,7 +7,7 @@ on: workflow_dispatch: env: - NODE_VERSION: 20 + NODE_VERSION: 22 PYTHON_VERSION: 3.14 jobs: @@ -67,6 +67,8 @@ jobs: - name: Build Docusaurus docs run: make build-docs + env: + APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} - name: Set up GitHub Pages uses: actions/configure-pages@v5 diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml index 91b43492..449349ba 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/pre_release.yaml @@ -32,18 +32,13 @@ jobs: release_type: prerelease existing_changelog_path: CHANGELOG.md - # If github.ref points to a [ci skip] commit, this workflow won't run. - # Otherwise, these checks will be launched from the `run_code_checks` workflow. - wait_for_checks: - name: Wait for code checks to pass - uses: apify/workflows/.github/workflows/wait_for_checks.yaml@wait-for-checks-action - with: - ref: ${{ github.ref }} - check-regexp: "( check| tests)$" + run_code_checks: + name: Run code checks + uses: ./.github/workflows/run_code_checks.yaml update_changelog: name: Update changelog - needs: [release_metadata, wait_for_checks] + needs: [release_metadata, run_code_checks] uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main with: version_number: ${{ needs.release_metadata.outputs.version_number }} @@ -53,7 +48,7 @@ jobs: publish_to_pypi: name: Publish to PyPI - needs: [release_metadata, wait_for_checks, update_changelog] + needs: [release_metadata, run_code_checks, update_changelog] runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index 051446c3..d7fbbaec 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -11,6 +11,9 @@ on: # It should also be possible to trigger checks manually workflow_dispatch: + # Allow this workflow to be invoked as a reusable workflow from other workflows + workflow_call: + jobs: actions_lint_check: name: Actions lint check From e5cbb35cd38bb2b0b3d279db6d08f2c7f7a90f6d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Sun, 14 Dec 2025 18:22:17 +0100 Subject: [PATCH 3/7] Separate code_checks and doc_checks --- .github/workflows/build_and_deploy_docs.yaml | 5 +++++ ...{run_code_checks.yaml => code_checks.yaml} | 4 ---- .github/workflows/doc_checks.yaml | 20 +++++++++++++++++++ .github/workflows/pre_release.yaml | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) rename .github/workflows/{run_code_checks.yaml => code_checks.yaml} (94%) create mode 100644 .github/workflows/doc_checks.yaml diff --git a/.github/workflows/build_and_deploy_docs.yaml b/.github/workflows/build_and_deploy_docs.yaml index e77a41a3..00c7c9f6 100644 --- a/.github/workflows/build_and_deploy_docs.yaml +++ b/.github/workflows/build_and_deploy_docs.yaml @@ -11,7 +11,12 @@ env: PYTHON_VERSION: 3.14 jobs: + doc_checks: + name: Doc checks + uses: ./.github/workflows/doc_checks.yaml + build_and_deploy_docs: + needs: [doc_checks] environment: name: github-pages permissions: diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/code_checks.yaml similarity index 94% rename from .github/workflows/run_code_checks.yaml rename to .github/workflows/code_checks.yaml index d7fbbaec..365d2186 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/code_checks.yaml @@ -50,10 +50,6 @@ jobs: name: Async dostrings check uses: ./.github/workflows/_async_docstrings_check.yaml - docs_check: - name: Docs check - uses: apify/workflows/.github/workflows/python_docs_check.yaml@main - integration_tests: name: Integration tests needs: [lint_check, type_check, unit_tests] diff --git a/.github/workflows/doc_checks.yaml b/.github/workflows/doc_checks.yaml new file mode 100644 index 00000000..f4e6615e --- /dev/null +++ b/.github/workflows/doc_checks.yaml @@ -0,0 +1,20 @@ +name: Run doc checks + +on: + # Trigger code checks on opening a new pull request. + # Secrets are only made available to the integration tests job, with a manual approval + # step required for PRs from forks. This prevents their potential exposure. + pull_request: + + # Trigger for pushing to the master branch is handled by the build-and-deploy-docs workflow. + + # It should also be possible to trigger checks manually + workflow_dispatch: + + # Allow this workflow to be invoked as a reusable workflow from other workflows + workflow_call: + +jobs: + doc_checks: + name: Run doc checks + uses: apify/workflows/.github/workflows/python_docs_check.yaml@main diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml index 449349ba..2132e85e 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/pre_release.yaml @@ -34,7 +34,7 @@ jobs: run_code_checks: name: Run code checks - uses: ./.github/workflows/run_code_checks.yaml + uses: ./.github/workflows/code_checks.yaml update_changelog: name: Update changelog From 273e64212db33ada579ebd3a66903346c48002e5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 15 Dec 2025 17:24:18 +0100 Subject: [PATCH 4/7] naming update --- .github/workflows/_check_code.yaml | 30 +++++++++ .github/workflows/_check_docs.yaml | 14 +++++ ...ings_check.yaml => _check_docstrings.yaml} | 12 ++-- ...nd_deploy_docs.yaml => _release_docs.yaml} | 40 ++++-------- .../{pre_release.yaml => _release_pre.yaml} | 30 ++++----- .github/workflows/_tests.yaml | 29 +++++++++ .github/workflows/check_pr_title.yaml | 14 ----- .github/workflows/code_checks.yaml | 62 ------------------- .github/workflows/doc_checks.yaml | 20 ------ ...elease.yaml => manual_release_stable.yaml} | 32 ++++------ .../{update_new_issue.yaml => on_issue.yaml} | 5 +- .github/workflows/on_master.yaml | 42 +++++++++++++ .github/workflows/on_pull_request.yaml | 32 ++++++++++ .pre-commit-config.yaml | 4 +- 14 files changed, 199 insertions(+), 167 deletions(-) create mode 100644 .github/workflows/_check_code.yaml create mode 100644 .github/workflows/_check_docs.yaml rename .github/workflows/{_async_docstrings_check.yaml => _check_docstrings.yaml} (70%) rename .github/workflows/{build_and_deploy_docs.yaml => _release_docs.yaml} (64%) rename .github/workflows/{pre_release.yaml => _release_pre.yaml} (76%) create mode 100644 .github/workflows/_tests.yaml delete mode 100644 .github/workflows/check_pr_title.yaml delete mode 100644 .github/workflows/code_checks.yaml delete mode 100644 .github/workflows/doc_checks.yaml rename .github/workflows/{release.yaml => manual_release_stable.yaml} (80%) rename .github/workflows/{update_new_issue.yaml => on_issue.yaml} (86%) create mode 100644 .github/workflows/on_master.yaml create mode 100644 .github/workflows/on_pull_request.yaml diff --git a/.github/workflows/_check_code.yaml b/.github/workflows/_check_code.yaml new file mode 100644 index 00000000..765a6a83 --- /dev/null +++ b/.github/workflows/_check_code.yaml @@ -0,0 +1,30 @@ +name: Code checks + +on: + # Runs when manually triggered from the GitHub UI. + workflow_dispatch: + + # Runs when invoked by another workflow. + workflow_call: + +jobs: + actions_lint_check: + name: Actions lint check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Run actionlint + uses: rhysd/actionlint@v1.7.9 + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + with: + python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + with: + python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' diff --git a/.github/workflows/_check_docs.yaml b/.github/workflows/_check_docs.yaml new file mode 100644 index 00000000..f9941233 --- /dev/null +++ b/.github/workflows/_check_docs.yaml @@ -0,0 +1,14 @@ +name: Doc checks + +on: + # Runs when manually triggered from the GitHub UI. + workflow_dispatch: + + # Runs when invoked by another workflow. + workflow_call: + +jobs: + doc_checks: + name: Doc checks + uses: apify/workflows/.github/workflows/python_docs_check.yaml@main + secrets: inherit diff --git a/.github/workflows/_async_docstrings_check.yaml b/.github/workflows/_check_docstrings.yaml similarity index 70% rename from .github/workflows/_async_docstrings_check.yaml rename to .github/workflows/_check_docstrings.yaml index d9e4bb1a..ba24c63d 100644 --- a/.github/workflows/_async_docstrings_check.yaml +++ b/.github/workflows/_check_docstrings.yaml @@ -1,14 +1,18 @@ -name: Async docstrings check +name: Docstrings checks on: + # Runs when manually triggered from the GitHub UI. + workflow_dispatch: + + # Runs when invoked by another workflow. workflow_call: env: PYTHON_VERSION: 3.14 jobs: - async_docstring_check: - name: Async docstrings check + docstrings_checks: + name: Docstrings checks runs-on: ubuntu-latest steps: @@ -28,5 +32,5 @@ jobs: - name: Install dependencies run: make install-dev - - name: Run async docstrings check + - name: Async docstrings check run: make check-async-docstrings diff --git a/.github/workflows/build_and_deploy_docs.yaml b/.github/workflows/_release_docs.yaml similarity index 64% rename from .github/workflows/build_and_deploy_docs.yaml rename to .github/workflows/_release_docs.yaml index 00c7c9f6..366a0758 100644 --- a/.github/workflows/build_and_deploy_docs.yaml +++ b/.github/workflows/_release_docs.yaml @@ -1,22 +1,23 @@ -name: Build and deploy docs +name: Doc release on: - push: - branches: - - master + # Runs when manually triggered from the GitHub UI. workflow_dispatch: + # Runs when invoked by another workflow. + workflow_call: + inputs: + ref: + required: true + type: string + env: NODE_VERSION: 22 PYTHON_VERSION: 3.14 jobs: - doc_checks: - name: Doc checks - uses: ./.github/workflows/doc_checks.yaml - - build_and_deploy_docs: - needs: [doc_checks] + release_docs: + name: Doc release environment: name: github-pages permissions: @@ -30,29 +31,12 @@ jobs: uses: actions/checkout@v6 with: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }} - name: Set up Node uses: actions/setup-node@v6 with: node-version: ${{ env.NODE_VERSION }} - cache: npm - cache-dependency-path: website/package-lock.json - - - name: Install Node dependencies - run: | - npm install - npm update @apify/docs-theme - working-directory: ./website - - # We do this as early as possible to prevent conflicts if someone else would push something in the meantime - - name: Commit the updated package.json and lockfile - run: | - git config user.name 'GitHub Actions' - git config user.email 'github-actions[bot]@users.noreply.github.com' - git add website/package.json - git add website/package-lock.json - git diff-index --quiet HEAD || git commit -m 'chore: Automatic docs theme update [skip ci]' || true - git push - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/_release_pre.yaml similarity index 76% rename from .github/workflows/pre_release.yaml rename to .github/workflows/_release_pre.yaml index 2132e85e..8562ada1 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/_release_pre.yaml @@ -1,16 +1,16 @@ -name: Create a pre-release +name: Pre-release on: - # Trigger a beta version release (pre-release) on push to the master branch. - push: - branches: - - master - tags-ignore: - - "**" # Ignore all tags to prevent duplicate builds when tags are pushed. - - # Or it can be triggered manually. + # Runs when manually triggered from the GitHub UI. workflow_dispatch: + # Runs when invoked by another workflow. + workflow_call: + outputs: + changelog_commitish: + description: The commit SHA after updating the changelog + value: ${{ jobs.update_changelog.outputs.changelog_commitish }} + concurrency: group: release cancel-in-progress: false @@ -32,13 +32,9 @@ jobs: release_type: prerelease existing_changelog_path: CHANGELOG.md - run_code_checks: - name: Run code checks - uses: ./.github/workflows/code_checks.yaml - update_changelog: name: Update changelog - needs: [release_metadata, run_code_checks] + needs: [release_metadata] uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main with: version_number: ${{ needs.release_metadata.outputs.version_number }} @@ -47,8 +43,9 @@ jobs: APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} publish_to_pypi: + if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs')" name: Publish to PyPI - needs: [release_metadata, run_code_checks, update_changelog] + needs: [release_metadata, update_changelog] runs-on: ubuntu-latest permissions: contents: write @@ -67,3 +64,6 @@ jobs: # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + + # TODO: add job for publish package to Conda + # https://github.com/apify/crawlee-python/issues/104 diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml new file mode 100644 index 00000000..4c609824 --- /dev/null +++ b/.github/workflows/_tests.yaml @@ -0,0 +1,29 @@ +name: Tests + +on: + # Runs when manually triggered from the GitHub UI. + workflow_dispatch: + + # Runs when invoked by another workflow. + workflow_call: + +jobs: + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + secrets: inherit + with: + python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + operating-systems: '["ubuntu-latest", "windows-latest"]' + python-version-for-codecov: "3.14" + operating-system-for-codecov: ubuntu-latest + + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit + with: + python-versions: '["3.10", "3.14"]' + operating-systems: '["ubuntu-latest"]' + python-version-for-codecov: "3.14" + operating-system-for-codecov: ubuntu-latest diff --git a/.github/workflows/check_pr_title.yaml b/.github/workflows/check_pr_title.yaml deleted file mode 100644 index 0867f08b..00000000 --- a/.github/workflows/check_pr_title.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Check PR title - -on: - pull_request_target: - types: [opened, edited, synchronize] - -jobs: - check_pr_title: - name: Check PR title - runs-on: ubuntu-latest - steps: - - uses: amannn/action-semantic-pull-request@v6.1.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/code_checks.yaml b/.github/workflows/code_checks.yaml deleted file mode 100644 index 365d2186..00000000 --- a/.github/workflows/code_checks.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: Run code checks - -on: - # Trigger code checks on opening a new pull request. - # Secrets are only made available to the integration tests job, with a manual approval - # step required for PRs from forks. This prevents their potential exposure. - pull_request: - - # Trigger for pushing to the master branch is handled by the pre-release workflow. - - # It should also be possible to trigger checks manually - workflow_dispatch: - - # Allow this workflow to be invoked as a reusable workflow from other workflows - workflow_call: - -jobs: - actions_lint_check: - name: Actions lint check - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Run actionlint - uses: rhysd/actionlint@v1.7.9 - - lint_check: - name: Lint check - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - - type_check: - name: Type check - uses: apify/workflows/.github/workflows/python_type_check.yaml@main - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - - unit_tests: - name: Unit tests - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - secrets: inherit - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - operating-systems: '["ubuntu-latest", "windows-latest"]' - python-version-for-codecov: "3.14" - operating-system-for-codecov: ubuntu-latest - - async_docstrings: - name: Async dostrings check - uses: ./.github/workflows/_async_docstrings_check.yaml - - integration_tests: - name: Integration tests - needs: [lint_check, type_check, unit_tests] - uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - secrets: inherit - with: - python-versions: '["3.10", "3.14"]' - operating-systems: '["ubuntu-latest"]' - python-version-for-codecov: "3.14" - operating-system-for-codecov: ubuntu-latest diff --git a/.github/workflows/doc_checks.yaml b/.github/workflows/doc_checks.yaml deleted file mode 100644 index f4e6615e..00000000 --- a/.github/workflows/doc_checks.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Run doc checks - -on: - # Trigger code checks on opening a new pull request. - # Secrets are only made available to the integration tests job, with a manual approval - # step required for PRs from forks. This prevents their potential exposure. - pull_request: - - # Trigger for pushing to the master branch is handled by the build-and-deploy-docs workflow. - - # It should also be possible to trigger checks manually - workflow_dispatch: - - # Allow this workflow to be invoked as a reusable workflow from other workflows - workflow_call: - -jobs: - doc_checks: - name: Run doc checks - uses: apify/workflows/.github/workflows/python_docs_check.yaml@main diff --git a/.github/workflows/release.yaml b/.github/workflows/manual_release_stable.yaml similarity index 80% rename from .github/workflows/release.yaml rename to .github/workflows/manual_release_stable.yaml index 4216e775..a567f904 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/manual_release_stable.yaml @@ -1,7 +1,7 @@ -name: Create a release +name: Stable release on: - # Trigger a stable version release via GitHub's UI, with the ability to specify the type of release. + # Runs when manually triggered from the GitHub UI, with options to specify the type of release. workflow_dispatch: inputs: release_type: @@ -26,8 +26,13 @@ concurrency: cancel-in-progress: false jobs: + code_checks: + name: Code checks + uses: ./.github/workflows/_check_code.yaml + release_metadata: name: Prepare release metadata + needs: [code_checks] runs-on: ubuntu-latest outputs: version_number: ${{ steps.release_metadata.outputs.version_number }} @@ -43,25 +48,9 @@ jobs: custom_version: ${{ inputs.custom_version }} existing_changelog_path: CHANGELOG.md - lint_check: - name: Lint check - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - - type_check: - name: Type check - uses: apify/workflows/.github/workflows/python_type_check.yaml@main - with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - - async_docstrings: - name: Async dostrings check - uses: ./.github/workflows/_async_docstrings_check.yaml - update_changelog: name: Update changelog - needs: [release_metadata, lint_check, type_check] + needs: [release_metadata] uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main with: version_number: ${{ needs.release_metadata.outputs.version_number }} @@ -70,7 +59,7 @@ jobs: APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} create_github_release: - name: Create github release + name: Create GitHub release needs: [release_metadata, update_changelog] runs-on: ubuntu-latest env: @@ -105,3 +94,6 @@ jobs: # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + + # TODO: add job for publish package to Conda + # https://github.com/apify/crawlee-python/issues/104 diff --git a/.github/workflows/update_new_issue.yaml b/.github/workflows/on_issue.yaml similarity index 86% rename from .github/workflows/update_new_issue.yaml rename to .github/workflows/on_issue.yaml index c0f35efd..c758a371 100644 --- a/.github/workflows/update_new_issue.yaml +++ b/.github/workflows/on_issue.yaml @@ -1,13 +1,14 @@ -name: Update new issue +name: CI (issue) on: + # Runs when a new issue is opened. issues: types: - opened jobs: label_issues: - name: Label issues + name: Add labels runs-on: ubuntu-latest permissions: issues: write diff --git a/.github/workflows/on_master.yaml b/.github/workflows/on_master.yaml new file mode 100644 index 00000000..fe27c6bc --- /dev/null +++ b/.github/workflows/on_master.yaml @@ -0,0 +1,42 @@ +name: CI (master) + +on: + # Runs on every push to the master branch. + push: + branches: + - master + tags-ignore: + - "**" # Ignore all tags to avoid duplicate executions triggered by tag pushes. + +jobs: + doc_checks: + name: Doc checks + uses: ./.github/workflows/_check_docs.yaml + secrets: inherit + + code_checks: + name: Code checks + uses: ./.github/workflows/_check_code.yaml + + docstrings_checks: + name: Docstrings checks + uses: ./.github/workflows/_check_docstrings.yaml + + tests: + name: Tests + uses: ./.github/workflows/_tests.yaml + secrets: inherit + + pre_release: + name: Pre-release + needs: [code_checks, tests] + uses: ./.github/workflows/_release_pre.yaml + secrets: inherit + + release_docs: + name: Doc release + needs: [doc_checks, pre_release] + uses: ./.github/workflows/_release_docs.yaml + with: + ref: ${{ needs.pre_release.outputs.changelog_commitish }} + secrets: inherit diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml new file mode 100644 index 00000000..b46e5526 --- /dev/null +++ b/.github/workflows/on_pull_request.yaml @@ -0,0 +1,32 @@ +name: CI (PR) + +on: + # Runs whenever a pull request is opened or updated. + pull_request: + +jobs: + check_pr_title: + name: Check PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v6.1.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + doc_checks: + name: Doc checks + uses: ./.github/workflows/_check_docs.yaml + secrets: inherit + + code_checks: + name: Code checks + uses: ./.github/workflows/_check_code.yaml + + docstrings_checks: + name: Docstrings checks + uses: ./.github/workflows/_check_docstrings.yaml + + tests: + name: Tests + uses: ./.github/workflows/_tests.yaml + secrets: inherit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 082c8f50..e909d818 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,8 +13,8 @@ repos: language: system pass_filenames: false - - id: async-docstrings-check - name: Async docstrings check + - id: docstrings-check + name: Docstrings check entry: make check-async-docstrings language: system pass_filenames: false From 80399749ca3ba3fb8eada47d8fc1907878e017fc Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 15 Dec 2025 17:43:05 +0100 Subject: [PATCH 5/7] rm todo to conda --- .github/workflows/_release_pre.yaml | 4 +--- .github/workflows/manual_release_stable.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_release_pre.yaml b/.github/workflows/_release_pre.yaml index 8562ada1..7fee938a 100644 --- a/.github/workflows/_release_pre.yaml +++ b/.github/workflows/_release_pre.yaml @@ -61,9 +61,7 @@ jobs: is_prerelease: "yes" version_number: ${{ needs.release_metadata.outputs.version_number }} ref: ${{ needs.update_changelog.outputs.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - - # TODO: add job for publish package to Conda - # https://github.com/apify/crawlee-python/issues/104 diff --git a/.github/workflows/manual_release_stable.yaml b/.github/workflows/manual_release_stable.yaml index a567f904..676f2e63 100644 --- a/.github/workflows/manual_release_stable.yaml +++ b/.github/workflows/manual_release_stable.yaml @@ -91,9 +91,7 @@ jobs: is_prerelease: "" version_number: ${{ needs.release_metadata.outputs.version_number }} ref: ${{ needs.update_changelog.outputs.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - - # TODO: add job for publish package to Conda - # https://github.com/apify/crawlee-python/issues/104 From c742c44bb0239d023aaf9e9e3786eb08d1d28072 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 15 Dec 2025 17:48:45 +0100 Subject: [PATCH 6/7] revert docs build and publish NPM-related changes --- .github/workflows/_release_docs.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/_release_docs.yaml b/.github/workflows/_release_docs.yaml index 366a0758..c322544c 100644 --- a/.github/workflows/_release_docs.yaml +++ b/.github/workflows/_release_docs.yaml @@ -37,6 +37,24 @@ jobs: uses: actions/setup-node@v6 with: node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: website/package-lock.json + + - name: Install Node dependencies + run: | + npm install + npm update @apify/docs-theme + working-directory: ./website + + # We do this as early as possible to prevent conflicts if someone else would push something in the meantime + - name: Commit the updated package.json and lockfile + run: | + git config user.name 'GitHub Actions' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git add website/package.json + git add website/package-lock.json + git diff-index --quiet HEAD || git commit -m 'chore: Automatic docs theme update [skip ci]' || true + git push - name: Set up Python uses: actions/setup-python@v6 From 2c80032369e188b7875a4746608ff807b1829809 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 16 Dec 2025 19:21:12 +0100 Subject: [PATCH 7/7] Update --- .github/workflows/_check_docs.yaml | 1 - .github/workflows/_release_docs.yaml | 3 +- .github/workflows/_release_pre.yaml | 67 ----------------- .github/workflows/manual_release_stable.yaml | 55 +++++++------- .github/workflows/on_master.yaml | 79 +++++++++++++++++--- .github/workflows/on_pull_request.yaml | 5 +- 6 files changed, 100 insertions(+), 110 deletions(-) delete mode 100644 .github/workflows/_release_pre.yaml diff --git a/.github/workflows/_check_docs.yaml b/.github/workflows/_check_docs.yaml index f9941233..5ea476d1 100644 --- a/.github/workflows/_check_docs.yaml +++ b/.github/workflows/_check_docs.yaml @@ -11,4 +11,3 @@ jobs: doc_checks: name: Doc checks uses: apify/workflows/.github/workflows/python_docs_check.yaml@main - secrets: inherit diff --git a/.github/workflows/_release_docs.yaml b/.github/workflows/_release_docs.yaml index c322544c..af809838 100644 --- a/.github/workflows/_release_docs.yaml +++ b/.github/workflows/_release_docs.yaml @@ -14,6 +14,7 @@ on: env: NODE_VERSION: 22 PYTHON_VERSION: 3.14 + CHECKOUT_REF: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }} jobs: release_docs: @@ -31,7 +32,7 @@ jobs: uses: actions/checkout@v6 with: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }} + ref: ${{ env.CHECKOUT_REF }} - name: Set up Node uses: actions/setup-node@v6 diff --git a/.github/workflows/_release_pre.yaml b/.github/workflows/_release_pre.yaml deleted file mode 100644 index 7fee938a..00000000 --- a/.github/workflows/_release_pre.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: Pre-release - -on: - # Runs when manually triggered from the GitHub UI. - workflow_dispatch: - - # Runs when invoked by another workflow. - workflow_call: - outputs: - changelog_commitish: - description: The commit SHA after updating the changelog - value: ${{ jobs.update_changelog.outputs.changelog_commitish }} - -concurrency: - group: release - cancel-in-progress: false - -jobs: - release_metadata: - if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci') && startsWith(github.repository, 'apify/')" - name: Prepare release metadata - runs-on: ubuntu-latest - outputs: - version_number: ${{ steps.release_metadata.outputs.version_number }} - tag_name: ${{ steps.release_metadata.outputs.tag_name }} - changelog: ${{ steps.release_metadata.outputs.changelog }} - steps: - - uses: apify/workflows/git-cliff-release@main - id: release_metadata - name: Prepare release metadata - with: - release_type: prerelease - existing_changelog_path: CHANGELOG.md - - update_changelog: - name: Update changelog - needs: [release_metadata] - uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main - with: - version_number: ${{ needs.release_metadata.outputs.version_number }} - changelog: ${{ needs.release_metadata.outputs.changelog }} - secrets: - APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - publish_to_pypi: - if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs')" - name: Publish to PyPI - needs: [release_metadata, update_changelog] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write # Required for OIDC authentication. - environment: - name: pypi - url: https://pypi.org/project/apify-client - steps: - - name: Prepare distribution - uses: apify/workflows/prepare-pypi-distribution@main - with: - package_name: apify-client - is_prerelease: "yes" - version_number: ${{ needs.release_metadata.outputs.version_number }} - ref: ${{ needs.update_changelog.outputs.changelog_commitish }} - - # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/manual_release_stable.yaml b/.github/workflows/manual_release_stable.yaml index 676f2e63..332a5cf6 100644 --- a/.github/workflows/manual_release_stable.yaml +++ b/.github/workflows/manual_release_stable.yaml @@ -30,52 +30,51 @@ jobs: name: Code checks uses: ./.github/workflows/_check_code.yaml - release_metadata: - name: Prepare release metadata + release_prepare: + name: Release prepare needs: [code_checks] runs-on: ubuntu-latest outputs: - version_number: ${{ steps.release_metadata.outputs.version_number }} - tag_name: ${{ steps.release_metadata.outputs.tag_name }} - changelog: ${{ steps.release_metadata.outputs.changelog }} - release_notes: ${{ steps.release_metadata.outputs.release_notes }} + version_number: ${{ steps.release_prepare.outputs.version_number }} + tag_name: ${{ steps.release_prepare.outputs.tag_name }} + changelog: ${{ steps.release_prepare.outputs.changelog }} + release_notes: ${{ steps.release_prepare.outputs.release_notes }} steps: - uses: apify/workflows/git-cliff-release@main - name: Prepare release metadata - id: release_metadata + name: Release prepare + id: release_prepare with: release_type: ${{ inputs.release_type }} custom_version: ${{ inputs.custom_version }} existing_changelog_path: CHANGELOG.md - update_changelog: - name: Update changelog - needs: [release_metadata] + changelog_update: + name: Changelog update + needs: [release_prepare] uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main with: - version_number: ${{ needs.release_metadata.outputs.version_number }} - changelog: ${{ needs.release_metadata.outputs.changelog }} - secrets: - APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + version_number: ${{ needs.release_prepare.outputs.version_number }} + changelog: ${{ needs.release_prepare.outputs.changelog }} + secrets: inherit - create_github_release: - name: Create GitHub release - needs: [release_metadata, update_changelog] + github_release: + name: GitHub release + needs: [release_prepare, changelog_update] runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Create release + - name: GitHub release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ needs.release_metadata.outputs.tag_name }} - name: ${{ needs.release_metadata.outputs.version_number }} - target_commitish: ${{ needs.update_changelog.outputs.changelog_commitish }} - body: ${{ needs.release_metadata.outputs.release_notes }} + tag_name: ${{ needs.release_prepare.outputs.tag_name }} + name: ${{ needs.release_prepare.outputs.version_number }} + target_commitish: ${{ needs.changelog_update.outputs.changelog_commitish }} + body: ${{ needs.release_prepare.outputs.release_notes }} - publish_to_pypi: - name: Publish to PyPI - needs: [release_metadata, update_changelog] + pypi_publish: + name: PyPI publish + needs: [release_prepare, changelog_update] runs-on: ubuntu-latest permissions: contents: write @@ -89,8 +88,8 @@ jobs: with: package_name: apify-client is_prerelease: "" - version_number: ${{ needs.release_metadata.outputs.version_number }} - ref: ${{ needs.update_changelog.outputs.changelog_commitish }} + version_number: ${{ needs.release_prepare.outputs.version_number }} + ref: ${{ needs.changelog_update.outputs.changelog_commitish }} # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI diff --git a/.github/workflows/on_master.yaml b/.github/workflows/on_master.yaml index fe27c6bc..b7aa5ecd 100644 --- a/.github/workflows/on_master.yaml +++ b/.github/workflows/on_master.yaml @@ -1,17 +1,30 @@ name: CI (master) on: - # Runs on every push to the master branch. push: branches: - master tags-ignore: - - "**" # Ignore all tags to avoid duplicate executions triggered by tag pushes. + - "**" # Ignore all tags to avoid duplicate executions triggered by tag pushes. + +concurrency: + group: release + cancel-in-progress: false jobs: doc_checks: name: Doc checks uses: ./.github/workflows/_check_docs.yaml + + doc_release: + # Skip this for non-docs commits and forks. + if: "startsWith(github.event.head_commit.message, 'docs') && startsWith(github.repository, 'apify/')" + name: Doc release + needs: [doc_checks] + uses: ./.github/workflows/_release_docs.yaml + with: + # Use the same ref as the one that triggered the workflow. + ref: ${{ github.ref }} secrets: inherit code_checks: @@ -23,20 +36,66 @@ jobs: uses: ./.github/workflows/_check_docstrings.yaml tests: + # Skip this for "ci" and "docs" commits. + if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs')" name: Tests uses: ./.github/workflows/_tests.yaml secrets: inherit - pre_release: - name: Pre-release - needs: [code_checks, tests] - uses: ./.github/workflows/_release_pre.yaml + release_prepare: + # Skip this for "ci", "docs" and "test" commits and for forks. + if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'test') && startsWith(github.repository, 'apify/')" + name: Release prepare + needs: [code_checks, docstrings_checks, tests] + runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.release_prepare.outputs.version_number }} + tag_name: ${{ steps.release_prepare.outputs.tag_name }} + changelog: ${{ steps.release_prepare.outputs.changelog }} + steps: + - uses: apify/workflows/git-cliff-release@main + id: release_prepare + name: Release prepare + with: + release_type: prerelease + existing_changelog_path: CHANGELOG.md + + changelog_update: + name: Changelog update + needs: [release_prepare] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_prepare.outputs.version_number }} + changelog: ${{ needs.release_prepare.outputs.changelog }} secrets: inherit - release_docs: - name: Doc release - needs: [doc_checks, pre_release] + pypi_publish: + name: PyPI publish + needs: [release_prepare, changelog_update] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/apify-client + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: apify-client + is_prerelease: "yes" + version_number: ${{ needs.release_prepare.outputs.version_number }} + ref: ${{ needs.changelog_update.outputs.changelog_commitish }} + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + doc_release_post_publish: + name: Doc release post publish + needs: [changelog_update, pypi_publish] uses: ./.github/workflows/_release_docs.yaml with: - ref: ${{ needs.pre_release.outputs.changelog_commitish }} + # Use the ref from the changelog update to include the updated changelog. + ref: ${{ needs.changelog_update.outputs.changelog_commitish }} secrets: inherit diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index b46e5526..ddd27697 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -5,8 +5,8 @@ on: pull_request: jobs: - check_pr_title: - name: Check PR title + pr_title_check: + name: PR title check runs-on: ubuntu-latest steps: - uses: amannn/action-semantic-pull-request@v6.1.1 @@ -16,7 +16,6 @@ jobs: doc_checks: name: Doc checks uses: ./.github/workflows/_check_docs.yaml - secrets: inherit code_checks: name: Code checks