From c46bee8427ce3e212a883092fd52936be2b4ec36 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 18 Jul 2026 10:07:44 +0200 Subject: [PATCH 1/2] Enable fetching offline documentation format builds from repo with GH Artifacts --- build_docs.py | 217 ++++++++++++++++++++++++------ check_versions.py | 2 +- tests/test_build_docs_versions.py | 1 + tests/test_fetch_artifacts.py | 156 +++++++++++++++++++++ 4 files changed, 335 insertions(+), 41 deletions(-) create mode 100644 tests/test_fetch_artifacts.py diff --git a/build_docs.py b/build_docs.py index 4305a68..f45dfcc 100755 --- a/build_docs.py +++ b/build_docs.py @@ -47,6 +47,7 @@ import dataclasses import datetime as dt import filecmp +import io import json import logging import logging.handlers @@ -58,6 +59,7 @@ import subprocess import sys import venv +import zipfile from bisect import bisect_left as bisect from contextlib import contextmanager, suppress from pathlib import Path @@ -621,6 +623,8 @@ class DocBuilder: log_directory: Path skip_cache_invalidation: bool theme: str + fetch_artifacts: bool + github_token: str | None @property def html_only(self) -> bool: @@ -648,8 +652,9 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: if self.build_meta.is_translation: self.clone_translation() if trigger_reason := self.should_rebuild(force_build): - self.build_venv() - self.build() + if not self.fetch_artifacts or self.includes_html: + self.build_venv() + self.build(http) self.copy_build_to_webroot(http) self.save_state( build_start=start_timestamp, @@ -699,7 +704,7 @@ def translation_branch(self) -> str: branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) return locate_nearest_version(branches, self.build_meta.version) - def build(self) -> None: + def build(self, http: urllib3.PoolManager) -> None: """Build this version/language doc.""" logging.info("Build start.") start_time = perf_counter() @@ -719,48 +724,171 @@ def build(self) -> None: maketarget = "autobuild-dev" else: maketarget = "autobuild-stable" - if self.html_only: + + # If we fetch artifacts, we don't compile non-html locally, so we force html-only. + # However, if we don't even include HTML (e.g. no-html), we don't run make at all! + if self.fetch_artifacts: + if self.includes_html: + maketarget += "-html" + else: + maketarget = None + elif self.html_only: maketarget += "-html" - logging.info("Running make %s", maketarget) - python = self.venv / "bin" / "python" - sphinxbuild = self.venv / "bin" / "sphinx-build" - blurb = self.venv / "bin" / "blurb" - if self.includes_html: - site_url = self.build_meta.url - # Define a tag to enable opengraph socialcards previews - # (used in Doc/conf.py and requires matplotlib) - sphinxopts += ( - "-t create-social-cards", - f"-D ogp_site_url={site_url}", + if maketarget is not None: + logging.info("Running make %s", maketarget) + python = self.venv / "bin" / "python" + sphinxbuild = self.venv / "bin" / "sphinx-build" + blurb = self.venv / "bin" / "blurb" + + if self.includes_html: + site_url = self.build_meta.url + # Define a tag to enable opengraph socialcards previews + # (used in Doc/conf.py and requires matplotlib) + sphinxopts += ( + "-t create-social-cards", + f"-D ogp_site_url={site_url}", + ) + + if self.build_meta.version_tuple < (3, 8): + # Disable CPython switchers, we handle them now: + text = (self.checkout / "Doc" / "Makefile").read_text( + encoding="utf-8" + ) + text = text.replace(" -A switchers=1", "") + (self.checkout / "Doc" / "Makefile").write_text( + text, encoding="utf-8" + ) + + self.setup_indexsidebar() + run_with_logging(( + "make", + "-C", + self.checkout / "Doc", + f"PYTHON={python}", + f"SPHINXBUILD={sphinxbuild}", + f"BLURB={blurb}", + f"VENVDIR={self.venv}", + f"SPHINXOPTS={' '.join(sphinxopts)}", + "SPHINXERRORHANDLING=", + maketarget, + )) + self.log_directory.mkdir(parents=True, exist_ok=True) + chgrp(self.log_directory, group=self.group, recursive=True) + if self.includes_html: + setup_switchers( + self.switchers_content, self.checkout / "Doc" / "build" / "html" + ) + + if self.fetch_artifacts and not self.html_only: + self.download_non_html_artifacts(http) + + logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) + + def download_non_html_artifacts(self, http: urllib3.PoolManager) -> None: + """Download non-HTML artifacts from GitHub actions and extract to dist.""" + token = ( + self.github_token + or os.environ.get("GITHUB_TOKEN") + or os.environ.get("GH_TOKEN") + ) + if not token: + raise ValueError( + "GitHub token is required to download actions artifacts. " + "Please set the GITHUB_TOKEN or GH_TOKEN environment variable, " + "or pass it via the --github-token argument." ) - if self.build_meta.version_tuple < (3, 8): - # Disable CPython switchers, we handle them now: - text = (self.checkout / "Doc" / "Makefile").read_text(encoding="utf-8") - text = text.replace(" -A switchers=1", "") - (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") - - self.setup_indexsidebar() - run_with_logging(( - "make", - "-C", - self.checkout / "Doc", - f"PYTHON={python}", - f"SPHINXBUILD={sphinxbuild}", - f"BLURB={blurb}", - f"VENVDIR={self.venv}", - f"SPHINXOPTS={' '.join(sphinxopts)}", - "SPHINXERRORHANDLING=", - maketarget, - )) - self.log_directory.mkdir(parents=True, exist_ok=True) - chgrp(self.log_directory, group=self.group, recursive=True) - if self.includes_html: - setup_switchers( - self.switchers_content, self.checkout / "Doc" / "build" / "html" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "docsbuild-scripts-fetch-artifacts", + "Authorization": f"Bearer {token}", + } + + artifacts = [] + page = 1 + per_page = 100 + max_pages = 5 + version = self.build_meta.version + language = self.build_meta.language + pattern = rf"^python-{re.escape(version)}(?:\.\d+)?-{re.escape(language)}-docs" + + logging.info("Searching for non-HTML artifacts for %s/%s", language, version) + + while page <= max_pages: + url = f"https://api.github.com/repos/m-aciek/python-docs-offline/actions/artifacts?per_page={per_page}&page={page}" + logging.debug("Fetching page %d of artifacts: %s", page, url) + response = http.request("GET", url, headers=headers) + if response.status != 200: + raise RuntimeError( + f"Failed to fetch artifacts list (page {page}): HTTP {response.status} - {response.data.decode('utf-8', errors='ignore')}" + ) + + try: + page_data = json.loads(response.data) + except Exception as e: + raise RuntimeError( + f"Failed to parse page {page} artifacts JSON: {e}" + ) from e + + page_artifacts = page_data.get("artifacts", []) + if not page_artifacts: + break + + artifacts.extend(page_artifacts) + page += 1 + + # Filter matching artifacts + matching = [] + for artifact in artifacts: + name = artifact.get("name", "") + if not re.match(pattern, name): + continue + # Skip HTML and logs + if "-html" in name or "-logs" in name: + continue + matching.append(artifact) + + if not matching: + raise RuntimeError( + f"No matching non-HTML artifacts found for {language}/{version}" ) - logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) + + # Sort matching artifacts by created_at descending so we get the newest first + matching.sort(key=lambda x: x.get("created_at", ""), reverse=True) + + # Keep only the latest for each unique name + latest_artifacts = {} + for artifact in matching: + name = artifact["name"] + if name not in latest_artifacts: + latest_artifacts[name] = artifact + + dist_dir = self.checkout / "Doc" / "dist" + if dist_dir.exists(): + logging.info("Cleaning existing dist directory: %s", dist_dir) + shutil.rmtree(dist_dir) + dist_dir.mkdir(parents=True, exist_ok=True) + + for name, artifact in latest_artifacts.items(): + artifact_id = artifact["id"] + download_url = artifact["archive_download_url"] + logging.info("Downloading artifact: %s (ID: %s)", name, artifact_id) + + resp = http.request("GET", download_url, headers=headers) + if resp.status != 200: + raise RuntimeError( + f"Failed to download artifact {name}: HTTP {resp.status} - {resp.data.decode('utf-8', errors='ignore')}" + ) + + try: + with zipfile.ZipFile(io.BytesIO(resp.data)) as zip_file: + zip_file.extractall(dist_dir) + logging.info("Successfully extracted %s to %s", name, dist_dir) + except Exception as e: + raise RuntimeError( + f"Failed to extract zip content for artifact {name}: {e}" + ) from e def build_venv(self) -> None: """Build a venv for the specific Python version. @@ -1105,6 +1233,15 @@ def parse_args() -> argparse.Namespace: "Builds all available languages by default.", metavar="fr", ) + parser.add_argument( + "--fetch-artifacts", + action="store_true", + help="Fetch non-HTML docs from GitHub Artifacts instead of building them locally.", + ) + parser.add_argument( + "--github-token", + help="GitHub Personal Access Token (PAT) for downloading artifacts. Can also be set via GITHUB_TOKEN or GH_TOKEN.", + ) parser.add_argument( "--version", action="store_true", diff --git a/check_versions.py b/check_versions.py index b50d69d..7253afd 100644 --- a/check_versions.py +++ b/check_versions.py @@ -32,7 +32,7 @@ def find_upstream_remote_name(repo: git.Repo) -> str: """Find a remote in the repo that matches the URL pattern.""" for remote in repo.remotes: for url in remote.urls: - if "github.com/python" in url: + if "github.com/python" in url or "github.com:python": return f"{remote.name}/" diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 8e7e230..ecb27e7 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -10,6 +10,7 @@ @pytest.fixture def versions() -> Versions: return Versions([ + Version(name="", status="planned", branch_or_tag=""), Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), diff --git a/tests/test_fetch_artifacts.py b/tests/test_fetch_artifacts.py new file mode 100644 index 0000000..6d03cfe --- /dev/null +++ b/tests/test_fetch_artifacts.py @@ -0,0 +1,156 @@ +import io +import json +import zipfile +from unittest import mock + +import pytest +import urllib3 + +from build_docs import BuildMetadata, DocBuilder, Language, Version + + +@pytest.fixture +def mock_http(): + return mock.Mock(spec=urllib3.PoolManager) + + +@pytest.fixture +def doc_builder(tmp_path): + version = Version(name="3.13", status="stable", branch_or_tag="3.13") + language = Language( + iso639_tag="fr", + name="French", + translated_name="français", + in_prod=True, + html_only=False, + sphinxopts=[], + ) + build_meta = BuildMetadata(_version=version, _language=language) + cpython_repo = mock.Mock() + + return DocBuilder( + build_meta=build_meta, + cpython_repo=cpython_repo, + docs_by_version_content=b"", + switchers_content=b"", + build_root=tmp_path, + www_root=tmp_path / "www", + select_output="no-html", + quick=False, + group="docs", + log_directory=tmp_path / "logs", + skip_cache_invalidation=True, + theme="python-docs-theme", + fetch_artifacts=True, + github_token="test-token", + ) + + +def create_mock_zip(): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as zf: + zf.writestr("python-3.13-fr-docs.epub", b"epub content") + zf.writestr("python-3.13-fr-docs-text.zip", b"text content") + return buffer.getvalue() + + +def test_download_non_html_artifacts_success(doc_builder, mock_http): + # Setup mock API responses + list_response_data = { + "artifacts": [ + # HTML artifact (should be skipped) + { + "id": 111, + "name": "python-3.13-fr-docs-html.zip", + "archive_download_url": "https://api.github.com/download/111", + "created_at": "2026-07-09T12:00:00Z", + }, + # Logs artifact (should be skipped) + { + "id": 222, + "name": "python-3.13-fr-pdf-logs.zip", + "archive_download_url": "https://api.github.com/download/222", + "created_at": "2026-07-09T12:00:00Z", + }, + # Matching older artifact (should be skipped in favor of 444) + { + "id": 333, + "name": "python-3.13-fr-docs.epub", + "archive_download_url": "https://api.github.com/download/333", + "created_at": "2026-07-09T11:00:00Z", + }, + # Matching newer artifact (should be downloaded) + { + "id": 444, + "name": "python-3.13-fr-docs.epub", + "archive_download_url": "https://api.github.com/download/444", + "created_at": "2026-07-09T13:00:00Z", + }, + # Other version (should be skipped) + { + "id": 555, + "name": "python-3.12-fr-docs.epub", + "archive_download_url": "https://api.github.com/download/555", + "created_at": "2026-07-09T12:00:00Z", + }, + ] + } + + mock_zip_content = create_mock_zip() + + def mock_request(method, url, headers=None, **kwargs): + if "artifacts" in url: + return urllib3.HTTPResponse( + body=json.dumps(list_response_data).encode("utf-8"), + status=200, + ) + elif "download/444" in url: + return urllib3.HTTPResponse( + body=mock_zip_content, + status=200, + ) + return urllib3.HTTPResponse(status=404) + + mock_http.request.side_effect = mock_request + + # Run execution + doc_builder.download_non_html_artifacts(mock_http) + + # Assertions + dist_dir = doc_builder.checkout / "Doc" / "dist" + assert dist_dir.exists() + assert (dist_dir / "python-3.13-fr-docs.epub").read_bytes() == b"epub content" + assert (dist_dir / "python-3.13-fr-docs-text.zip").read_bytes() == b"text content" + + # Verify requests were made + assert mock_http.request.call_count >= 2 + + +def test_download_non_html_artifacts_no_token(doc_builder, mock_http, monkeypatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + doc_builder.github_token = None + + with pytest.raises(ValueError, match="GitHub token is required"): + doc_builder.download_non_html_artifacts(mock_http) + + +def test_download_non_html_artifacts_no_matching(doc_builder, mock_http): + list_response_data = {"artifacts": []} + mock_http.request.return_value = urllib3.HTTPResponse( + body=json.dumps(list_response_data).encode("utf-8"), + status=200, + ) + + with pytest.raises(RuntimeError, match="No matching non-HTML artifacts found"): + doc_builder.download_non_html_artifacts(mock_http) + + +def test_download_non_html_artifacts_api_error(doc_builder, mock_http): + mock_http.request.return_value = urllib3.HTTPResponse( + body=b"Internal Server Error", + status=500, + ) + + with pytest.raises(RuntimeError, match="Failed to fetch artifacts list"): + doc_builder.download_non_html_artifacts(mock_http) From 4803fa9b346808d3edb819159dddce8efabbb7b3 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Sat, 18 Jul 2026 11:01:44 +0200 Subject: [PATCH 2/2] Update check_versions.py Co-authored-by: Zachary Ware --- check_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_versions.py b/check_versions.py index 7253afd..d34aa48 100644 --- a/check_versions.py +++ b/check_versions.py @@ -32,7 +32,7 @@ def find_upstream_remote_name(repo: git.Repo) -> str: """Find a remote in the repo that matches the URL pattern.""" for remote in repo.remotes: for url in remote.urls: - if "github.com/python" in url or "github.com:python": + if "github.com/python" in url or "github.com:python" in url: return f"{remote.name}/"