diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index d2c2a0d6..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,28 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - groups: - actions: - patterns: - - "*" - commit-message: - prefix: "chore" - - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "weekly" - groups: - dev-dependencies: - patterns: - - "*" - commit-message: - prefix: "chore" diff --git a/README.md b/README.md index 0bf5a5ca..25e2919d 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,14 @@ You can see the template in action in the [example project](https://github.com/D ## Create a new project from the commandline -You will need to `pip install copier` inside an activated `venv` from python3.11 or later, then you can create a new module via: +We recommend that you invoke copier via `uvx`, which will download, install, and run it in its own isolated `venv`. At Diamond you can `module load uv` to get `uvx` on your path, then: ``` git init --initial-branch=main /path/to/my-project # $_ resolves to /path/to/my-project -copier copy https://github.com/DiamondLightSource/python-copier-template.git $_ +uvx copier copy https://github.com/DiamondLightSource/python-copier-template.git $_ ``` -You can also use it via `uvx copier` if you have `uv` installed. - See https://DiamondLightSource.github.io/python-copier-template for more detailed documentation. diff --git a/docs/how-to.md b/docs/how-to.md index 27c9aca2..04b0eae9 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -6,6 +6,7 @@ Practical step-by-step guides for the more experienced user. :maxdepth: 1 :glob: +how-to/setup-repo how-to/dev-install how-to/* ``` diff --git a/docs/how-to/build-docs.md b/docs/how-to/build-docs.md index a943dad2..2351d0e5 100644 --- a/docs/how-to/build-docs.md +++ b/docs/how-to/build-docs.md @@ -37,3 +37,12 @@ If you want to watch additional directories for changes you can pass these as ar ``` $ tox -e docs-autobuild -- --watch tests ``` + +(building-docs-in-ci)= +## Building docs in CI + +After a successful run of CI + +Settings > Pages + +![Setup GitHub Pages](../images/gh-pages-setup.png) diff --git a/docs/how-to/coverage.md b/docs/how-to/coverage.md index 161ffc2c..af43cf91 100644 --- a/docs/how-to/coverage.md +++ b/docs/how-to/coverage.md @@ -3,6 +3,20 @@ Code coverage is reported to the command line and to a `cov.xml` file by the command `tox -e tests`. The file is uploaded to the Codecov service in CI. -## Adding a Codecov Token +(installing-codecov-github-app)= +## Installing Codecov GitHub app -If the repo is not hosted in DiamondLightSource, then you need to visit `https://app.codecov.io/account/gh//org-upload-token` to generate a token for your org, and store it as a secret named `CODECOV_TOKEN` in `https://github.com/organizations//settings/secrets/actions` +If your repo is hosted in the DiamondLightSource org, then the codecov GitHub app is already installed and a global token stored so you don't need to do anything. + +If your repo is in an org where the codecov GitHub app is not installed, then follow these steps to install it on the org: + +- Visit https://github.com/apps/codecov +- Click `Configure` +- Select the org your repo is hosted in +- Select `All repositories` and click `Install` + +![Install Codecov](../images/gh-install-codecov.png) + +Now you need to visit `https://app.codecov.io/account/gh//org-upload-token` to generate a token for your org, and store it as a secret named `CODECOV_TOKEN` in `https://github.com/organizations//settings/secrets/actions` + +Next time you create a pull request or merge to main, the code coverage will be uploaded, and the badge on the repository updated. diff --git a/docs/how-to/lock-requirements.md b/docs/how-to/lock-requirements.md index a99824d0..59aa0fef 100644 --- a/docs/how-to/lock-requirements.md +++ b/docs/how-to/lock-requirements.md @@ -28,6 +28,7 @@ To update all dependencies to their latest versions run: ``` uv sync --upgrade ``` +This command will be run by [renovate](./renovate) once a week in CI. ```{seealso} [The uv docs on locking and syncing](https://docs.astral.sh/uv/concepts/projects/sync) diff --git a/docs/how-to/pypi.md b/docs/how-to/pypi.md index 5cc9eaaf..ceacdc3b 100644 --- a/docs/how-to/pypi.md +++ b/docs/how-to/pypi.md @@ -14,7 +14,7 @@ You will need the following information: ## If publishing to the DiamondLightSource PyPI organisation -If you are publishing to the DiamondLightSource PyPI organisation then use the above information and follow the [Developer Portal Guide on PyPI publishing](https://dev-portal.diamond.ac.uk/guide/python/how-tos/pypi/). +If you are publishing to the DiamondLightSource PyPI organisation then use the above information and follow the [Developer Portal Guide on PyPI publishing](https://dev-guide.diamond.ac.uk/python/how-tos/pypi/). ## If publishing the PyPI project to another organisation diff --git a/docs/how-to/renovate.md b/docs/how-to/renovate.md new file mode 100644 index 00000000..5096fd38 --- /dev/null +++ b/docs/how-to/renovate.md @@ -0,0 +1,39 @@ +# Use renovate to keep dependencies up to date + +[Renovate](https://github.com/apps/renovate) is a tool for automating dependency updates. It is used to: +- Automatically update the [uv lock file](./lock-requirements.md) and automerge if tests pass +- Update any dependencies in the Dockerfile +- Update the versions of any GitHub actions not manager by the python-copier-template + +## Install the Renovate GitHub app + +The renovate app will: +- Create weekly PRs to update the lockfile to the latest versions, automerging if tests pass +- Create PRs for other dependencies whenever they are released, but not automerge + +If your repo is hosted in the DiamondLightSource org, then the Renovate GitHub app is already installed so you don't need to do anything. + +If your repo is in an org where the Renovate GitHub app is not installed, then follow these steps to install it on the org: + +- Visit https://github.com/apps/renovate +- Click `Configure` +- Select the org your repo is hosted in +- Select `All repositories` and click `Install` +- Select `Renovate Only` as the product +- Select `Scan only` as the mode +- Click `Finish` +- Click `Settings` +- Select the `Dependencies` tab +- Uncheck `Silent mode`, uncheck `Create onboarding PRs`, and click `Save` +- Visit https://github.com/apps/renovate-approve +- Click `Configure` +- Select the org your repo is hosted in +- Select `All repositories` and click `Install` + +![Install Renovate](../images/gh-install-renovate.png) + +Renovate will now run periodically to check for updates. + +## Pin the dependency dashboard issue + +Renovate will create a dependency dashboard when it first runs, this is the way you interact with the bot, requesting it to re-run. It is recommended that you pin this issue to your repo so you can find it easily. Select `Pin issue` in the right hand bar of the issue to do this. diff --git a/docs/how-to/setup-repo.md b/docs/how-to/setup-repo.md new file mode 100644 index 00000000..cd8dfc5b --- /dev/null +++ b/docs/how-to/setup-repo.md @@ -0,0 +1,35 @@ +# Setup the repository with recommended settings + +When the repository has been created, it will require some settings to be changed for all the features of the python-copier-template to work. These are listed below. + +## Discussions + +The contributing guide for the template recommends that new users start a discussion for questions. You can enable this feature by navigating to your repository page and: + +- Visiting `Settings` > `General` +- Scrolling down to `Features` +- Enabling it as shown + +![Setup GitHub Features](../images/gh-features-setup.png) + +## Branch protection + +GitHub will prompt you that your `main` branch is not protected. It is recommended that a [branch protection rule](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/managing-a-branch-protection-rule) is setup for `main` to require pull requests. + +If the repository is in the `DiamondLightSource` org, then follow [the dev guide](https://dev-guide.diamond.ac.uk/version-control/how-tos/github-setup-prs/) to see the recommended settings. + +## Code Coverage + +To ensure that code coverage is correctly uploaded to codecov.io follow [](#installing-codecov-github-app) + +## Renovate + +To ensure that your dependencies are kept up to date follow [](./renovate.md) + +## GitHub pages + +If you configured the project to have sphinx docs then follow [](#building-docs-in-ci) + +## PyPI uploading + +If you configured the project to upload built wheels to PyPI then follow [](./pypi.md) diff --git a/docs/images/gh-features-setup.png b/docs/images/gh-features-setup.png new file mode 100644 index 00000000..ff8f1e4d Binary files /dev/null and b/docs/images/gh-features-setup.png differ diff --git a/docs/images/gh-install-codecov.png b/docs/images/gh-install-codecov.png new file mode 100644 index 00000000..b9c811a5 Binary files /dev/null and b/docs/images/gh-install-codecov.png differ diff --git a/docs/images/gh-install-renovate.png b/docs/images/gh-install-renovate.png new file mode 100644 index 00000000..a3690104 Binary files /dev/null and b/docs/images/gh-install-renovate.png differ diff --git a/docs/images/gh-pages-setup.png b/docs/images/gh-pages-setup.png new file mode 100644 index 00000000..db083b7e Binary files /dev/null and b/docs/images/gh-pages-setup.png differ diff --git a/docs/tutorials/adopt-existing.md b/docs/tutorials/adopt-existing.md index 7829ba0a..c8c5491d 100644 --- a/docs/tutorials/adopt-existing.md +++ b/docs/tutorials/adopt-existing.md @@ -18,11 +18,11 @@ Copier will *overwrite* files with the template files. Please check the changes If you have a [python3-pip-skeleton](https://github.com/DiamondLightSource/python3-pip-skeleton) based project then it is best to adopt the `1.0.0` release of this template, then `copier update` to get to the latest. This is because `copier update` will try and merge file changes across renames done between releases, while `copier copy` cannot. This looks like: ```shell -copier copy https://github.com/DiamondLightSource/python-copier-template.git --trust --vcs-ref=1.0.0 /path/to/existing-project +uvx copier copy https://github.com/DiamondLightSource/python-copier-template.git --trust --vcs-ref=1.0.0 /path/to/existing-project git diff # Examine the changes, put back anything you want to keep git commit -m "Adopt python-copier-template 1.0.0" -copier update /path/to/existing-project --trust +uvx copier update /path/to/existing-project --trust git diff # Examine the changes, resolve any merge conflicts git commit -m "Update to python-copier-template x.x.x" @@ -33,7 +33,7 @@ git commit -m "Update to python-copier-template x.x.x" If you have a project with a different structure then it is best to go straight to the latest release: ```shell -copier copy https://github.com/DiamondLightSource/python-copier-template.git /path/to/existing-project +uvx copier copy https://github.com/DiamondLightSource/python-copier-template.git /path/to/existing-project git diff # Examine the changes, put back anything you want to keep git commit -m "Adopt python-copier-template x.x.x" @@ -45,4 +45,4 @@ Copier does not touch any already existing files that do not conflict with the o ## Getting started with your new structure -You can now [](../how-to/dev-install), and then follow some of the other [](../how-to). +You can now [](../how-to/setup-repo), [](../how-to/dev-install), and then follow some of the other [](../how-to). diff --git a/docs/tutorials/create-new.md b/docs/tutorials/create-new.md index 2dd6095f..cef038e0 100644 --- a/docs/tutorials/create-new.md +++ b/docs/tutorials/create-new.md @@ -3,8 +3,9 @@ Once you have followed the [](./installation) tutorial, you can use `copier` to make a new project from the template: ``` -$ git init --initial-branch=main /path/to/my-project -$ copier copy https://github.com/DiamondLightSource/python-copier-template.git $_ +git init --initial-branch=main /path/to/my-project +# $_ resolves to /path/to/my-project +uvx copier copy https://github.com/DiamondLightSource/python-copier-template.git $_ ``` This will: @@ -16,27 +17,22 @@ This will: ## Committing the results -You can now check what the template has created, tweak the results if desired, and commit the results: -``` +You can now check what the template has created, tweak the results if desired, [](../how-to/lock-requirements), and commit the results: +```shell $ cd /path/to/my-project +$ uv sync $ git add . $ git commit -m "Expand from python-copier-template x.x.x" ``` ## Uploading to GitHub -If the project is to be under the DiamondLightSource organisation, request a new repository in the [#github-requests Slack channel](https://diamondlightsource.slack.com/archives/C06A18ZPP44) with the given repo name, description and who to add as an owner in addition to yourself. This will be replaced by [a template in the developer-portal in future.](https://github.com/DiamondLightSource/python-copier-template/issues/274) - -Else, you can now [create a new blank project on GitHub](https://github.com/new). Choose the same GitHub owner and repo name that you answered in the questions earlier. GitHub will now give you the commands needed to upload your repo from GitHub. - -## Settings +You can now [create a new blank project on GitHub](https://github.com/new). Choose the same GitHub owner, repo name and description that you answered in the questions earlier. GitHub will now give you the commands needed to upload your repo from GitHub. -You can now go to the cogwheel on the main page next to the "About" header, and set the project description to match the answer you gave. - -Then go to the `Settings` tab and set: - -- Enable Pages if you chose to use sphinx for your documentation +```{note} +At present you cannot make a project directly in the DiamondLightSource organisation, it must be made as a personal repository then [transferred in](https://dev-guide.diamond.ac.uk/version-control/how-tos/github-transfer-repo/). When it has been transferred then you can `copier update --vcs-ref=:current:` to re-answer the questions with the new org +``` ## Getting started with your new repo -You can now [](../how-to/dev-install), and then follow some of the other [](../how-to). +You can now [](../how-to/setup-repo), [](../how-to/dev-install), and then follow some of the other [](../how-to). diff --git a/docs/tutorials/installation.md b/docs/tutorials/installation.md index 432ed2d8..70b36988 100644 --- a/docs/tutorials/installation.md +++ b/docs/tutorials/installation.md @@ -3,45 +3,18 @@ This tutorial will take you through installing copier, the templating engine that will allow you to create new projects from the template, update existing projects in line with it, and keep projects in sync with changes to it. -## Check your version of python +## Install uv -You will need python 3.11 or later. You can check your version of python by -typing into a terminal: +We recommend that you invoke copier via `uvx`, which will download, install, and run it in its own isolated `venv`. -``` -$ python3 --version -``` +At Diamond you can `module load uv` to get `uvx` on the path. -:::{note} -At Diamond you can use `module load python` to get a more recent version of python on your path -::: +Otherwise please follow the [uv installation instructions](https://docs.astral.sh/uv/getting-started/installation). -## Create a virtual environment -It is recommended that you install into a “virtual environment” so this -installation will not interfere with any existing Python software: +## Try it out -``` -$ python3 -m venv /path/to/venv -$ source /path/to/venv/bin/activate -``` - -:::{note} -You may wish to deactivate any existing virual environments before sourcing the new -environment. Deactivation can be performed by executing: - -- `conda deactivate` for conda -- `deactivate` for venv or virtualenv -- `exit` for pipenv -::: - -## Installing copier - -You can now use `pip` to install copier so you can make your project from the template: - -``` -$ python3 -m pip install copier -``` +If you run `uvx copier --version` then `copier` will be downloaded, installed, and run, and will print its version. ## Conclusion diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..dc902a2d --- /dev/null +++ b/renovate.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ], + "lockFileMaintenance": { + "description": "Keep uv.lock up to date, merging if tests pass", + "enabled": true, + "automerge": true + }, + "pre-commit": { + "enabled": true + }, + "packageRules": [ + { + "description": "Disable python version as that is managed by python-copier-template", + "matchManagers": [ + "pyenv" + ], + "enabled": false + }, + { + "description": "Group non-major github action updates", + "groupName": "GitHub Actions", + "matchUpdateTypes": [ + "patch", + "minor" + ], + "matchManagers": [ + "github-actions" + ] + } + ] +} diff --git a/template/renovate.json.jinja b/template/renovate.json.jinja new file mode 100644 index 00000000..016ea7ae --- /dev/null +++ b/template/renovate.json.jinja @@ -0,0 +1,41 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ], + "lockFileMaintenance": { + "description": "Keep uv.lock up to date, merging if tests pass", + "enabled": true, + "automerge": true + }, + "packageRules": [ + { + "description": "Disable python version as that is managed by python-copier-template", + "matchManagers": [ + "pyenv" + ], + "enabled": false + }, + { + "description": "Disable github actions that are managed by python-copier-template", + "matchPackageNames": [ + "actions/checkout", + "astral-sh/setup-uv", + "actions/upload-artifact", + "actions/download-artifact", + "softprops/action-gh-release", + "codecov/codecov-action"{% if docker %}, + "docker/setup-buildx-action", + "docker/login-action", + "docker/build-push-action", + "docker/metadata-action"{% endif %}{% if pypi %}, + "pypa/gh-action-pypi-publish"{% endif %}{% if sphinx %}, + "peaceiris/actions-gh-pages"{% endif %} + ], + "matchManagers": [ + "github-actions" + ], + "enabled": false + } + ] +} diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" deleted file mode 120000 index 8a6fa73a..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" +++ /dev/null @@ -1 +0,0 @@ -./../../.github/dependabot.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" index 7674c4b3..5715df18 100644 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" @@ -49,7 +49,8 @@ jobs: {% endraw %}{% endif %}{% endif %}{% if sphinx %} docs: uses: ./.github/workflows/_docs.yml - + permissions: + contents: write {% endif %} dist: uses: ./.github/workflows/_dist.yml diff --git a/tests/test_example.py b/tests/test_example.py index 3de6a826..4b149e6b 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -1,6 +1,8 @@ import functools +import json import shlex import subprocess +import tomllib from collections.abc import Callable from pathlib import Path @@ -327,3 +329,60 @@ def test_catalog_info(tmp_path: Path): "owner": "group:default/daq", }, } + + +@pytest.mark.parametrize( + "override", + [ + {}, + {"docker": True}, + {"docker": True, "docker_debug": True}, + {"pypi": True}, + {"docs_type": "sphinx"}, + ], +) +def test_renovate_actions_match_what_is_shipped(override: dict, tmp_path: Path): + # Generate a project with the given answers + answers = { + "docker": False, + "docker_debug": False, + "pypi": False, + "docs_type": "README", + } + answers.update(override) + copy_project(tmp_path, **answers) + # Find the GitHub actions ignored by renovate + renovate_config_path = tmp_path / "renovate.json" + renovate_config = json.loads(renovate_config_path.read_text()) + config_github_actions = set(renovate_config["packageRules"][1]["matchPackageNames"]) + # Find the GitHub actions actually used in the workflows + used_github_actions = set[str]() + for workflow_file in (tmp_path / ".github" / "workflows").glob("*.yml"): + workflow = yaml.safe_load(workflow_file.read_text()) + for job in workflow.get("jobs", {}).values(): + for step in job.get("steps", []): + action = step.get("uses") + if action: + used_github_actions.add(action.split("@")[0]) + # Check they match + assert used_github_actions == config_github_actions + + +def test_python_versions_match(tmp_path: Path): + copy_project(tmp_path) + # Grab the python versions from ci.yml + ci_yaml = tmp_path / ".github" / "workflows" / "ci.yml" + workflow = yaml.safe_load(ci_yaml.read_text()) + python_versions = workflow["jobs"]["test"]["strategy"]["matrix"]["python-version"] + # Check .python-version is the first of these + python_version_file = tmp_path / ".python-version" + min_version = python_version_file.read_text().strip() + assert python_versions[0] == min_version + # Check pyproject.toml has correct requires-python and classifiers + pyproject_toml = tomllib.loads((tmp_path / "pyproject.toml").read_text()) + assert pyproject_toml["project"]["requires-python"] == f">={min_version}" + for version in python_versions: + assert ( + f"Programming Language :: Python :: {version}" + in pyproject_toml["project"]["classifiers"] + )