diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 34e26f5c2abb7..49fea6ed766a8 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -228,6 +228,9 @@ jobs: - task: multitool name: Multitool run: ./go update_multitool + - task: cddl + name: CDDL Spec Files + run: ./go update_cddl - task: authors name: Authors run: ./go authors @@ -282,6 +285,7 @@ jobs: } apply_patch versions "bump versions" apply_patch devtools "update devtools versions" + apply_patch cddl "update cddl spec files" apply_patch dep-updates "update dependencies" apply_patch changelogs "changelogs updated" apply_patch rust-changelogs "rust changelogs updated" @@ -313,6 +317,7 @@ jobs: |-----------|--------| | Versions | ${{ steps.apply.outputs.versions == 'true' && '✅ Updated' || '⏭️ Skipped (no changes)' }} | | CDP version | ${{ needs.parse-tag.outputs.language != 'all' && 'N/A' || (steps.apply.outputs.devtools == 'true' && '✅ Updated' || '⏭️ Skipped (no changes)') }} | + | CDDL spec files | ${{ needs.parse-tag.outputs.language != 'all' && 'N/A' || (steps.apply.outputs.cddl == 'true' && '✅ Updated' || '⏭️ Skipped (no changes)') }} | | Binding Dependencies | ${{ steps.apply.outputs.dep-updates == 'true' && '✅ Updated' || '⏭️ Skipped (no changes)' }} | | Changelogs | ${{ steps.apply.outputs.changelogs == 'true' && '✅ Updated' || '⏭️ Skipped (no changes)' }} | | Rust Changelogs | ${{ needs.parse-tag.outputs.language != 'all' && 'N/A' || (steps.apply.outputs.rust-changelogs == 'true' && '✅ Updated' || '⏭️ Skipped (no changes)') }} | diff --git a/MODULE.bazel b/MODULE.bazel index 0bdddacfb78e6..834c35ba74523 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -451,22 +451,11 @@ webref_cddl_extension = use_extension("//common:webref_cddl.bzl", "webref_cddl_e use_repo( webref_cddl_extension, "at_driver_all_cddl", - "at_driver_local_cddl", - "at_driver_remote_cddl", "permissions_all_cddl", - "permissions_local_cddl", - "permissions_remote_cddl", "prefetch_all_cddl", - "prefetch_local_cddl", "ua_client_hints_all_cddl", - "ua_client_hints_local_cddl", - "ua_client_hints_remote_cddl", "web_bluetooth_all_cddl", - "web_bluetooth_local_cddl", - "web_bluetooth_remote_cddl", "webdriver_bidi_all_cddl", - "webdriver_bidi_local_cddl", - "webdriver_bidi_remote_cddl", ) pin_browsers_extension = use_extension("//common:repositories.bzl", "pin_browsers_extension") diff --git a/Rakefile b/Rakefile index 18872ac755b3c..0c2b333162a6b 100644 --- a/Rakefile +++ b/Rakefile @@ -81,6 +81,12 @@ task :release_update do |_task, _arguments| Rake::Task[:update_multitool].invoke end +desc 'Update pinned CDDL spec files from w3c/webref' +task :update_cddl do |_task, _arguments| + puts 'Updating pinned CDDL spec references' + Bazel.execute('run', [], '//scripts:update_cddl') +end + desc 'Update Chrome DevTools support' task :update_cdp, [:channel] do |_task, arguments| chrome_channel = arguments[:channel] || 'stable' @@ -112,6 +118,7 @@ task :release_updates, [:tag, :channel] do |_task, arguments| if parsed[:patch].zero? Rake::Task['update_browsers'].invoke(arguments[:channel]) Rake::Task['update_cdp'].invoke(arguments[:channel]) + Rake::Task['update_cddl'].invoke Rake::Task['update_manager'].invoke Rake::Task['update_multitool'].invoke Rake::Task['authors'].invoke diff --git a/common/webref_cddl.bzl b/common/webref_cddl.bzl index 75d1eb202af2d..3091485f37fdb 100644 --- a/common/webref_cddl.bzl +++ b/common/webref_cddl.bzl @@ -2,31 +2,26 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +# Pinned to a single w3c/webref "main" commit (kept on main so pin-to-pin diffs +# stay auditable) and regenerated by the //scripts:update_cddl Bazel target: +# https://github.com/w3c/webref/tree/main/ed/cddl _COMMIT = "5301ec52f85f9156ae696f24523d722193fc0817" _BASE_URL = "https://raw.githubusercontent.com/w3c/webref/{commit}/ed/cddl".format(commit = _COMMIT) -# All CDDL files in https://github.com/w3c/webref/tree/{commit}/ed/cddl -# Each entry is (repo_name, filename, sha256). -# Files are downloaded as "spec.cddl" in each repo, so consumers can reference -# them consistently as @//file:spec.cddl. +# The "-all" (union) CDDL file for each protocol in +# https://github.com/w3c/webref/tree/{commit}/ed/cddl -- the per-end +# "-local"/"-remote" splits are skipped since generation merges the union. +# Which of these actually feed generation is selected in the py/ and +# javascript/ BUILD.bazel merge lists (this pins a superset). +# Each entry is (repo_name, filename, sha256). Files are downloaded as +# "spec.cddl" in each repo, referenced as @//file:spec.cddl. _CDDL_FILES = [ ("at_driver_all_cddl", "at-driver-all.cddl", "f8502ca866e494d9c46a5209eb0cfec57107fffe587d154a365a19e8b93dd7aa"), - ("at_driver_local_cddl", "at-driver-local-cddl.cddl", "f1b64b2d852c5ea826cc9a6431196979cdf7c73a0182c98dc7c3c40005bdbcba"), - ("at_driver_remote_cddl", "at-driver-remote-cddl.cddl", "18ec6dc05d515b6f01169db3aa27f36c38f3ecb0c7c070735d27c7f1b1957533"), ("permissions_all_cddl", "permissions-all.cddl", "50e9b0017415e27a18a190bf37df048d4513f8432e42fe97901c9f2d55204b50"), - ("permissions_local_cddl", "permissions-local-cddl.cddl", "0f9f266f9e991eb7656848e917318b5aae4b06b4cf3b48e95ee50ed264912cb8"), - ("permissions_remote_cddl", "permissions-remote-cddl.cddl", "17b968bbaf97782908c69637cc1152d0119c559f5f50b047e5e30894bf798983"), ("prefetch_all_cddl", "prefetch-all.cddl", "51409b998176a81f681252f8ee16bea5a54a3be9d1cfee9f13ca34efd1feb5ea"), - ("prefetch_local_cddl", "prefetch-local-cddl.cddl", "51409b998176a81f681252f8ee16bea5a54a3be9d1cfee9f13ca34efd1feb5ea"), ("ua_client_hints_all_cddl", "ua-client-hints-all.cddl", "6bb41f05d09c755305226b7350970e54f6404698510ea2e1e7a931eaa2647aeb"), - ("ua_client_hints_local_cddl", "ua-client-hints-local-cddl.cddl", "c71077e4ecea5f1cbed309cfb16418195ffdb631621eff3f9c784907c9c2c7dc"), - ("ua_client_hints_remote_cddl", "ua-client-hints-remote-cddl.cddl", "c62185206132fbea3085e87af93315a7ecc39c802c20a2aa5d808d668e5664f7"), ("web_bluetooth_all_cddl", "web-bluetooth-all.cddl", "bc687d19c1e92cf4f0a37bbf7f9aa21db956344b34d8cdda4319b7f4615b6d1d"), - ("web_bluetooth_local_cddl", "web-bluetooth-local-cddl.cddl", "bc687d19c1e92cf4f0a37bbf7f9aa21db956344b34d8cdda4319b7f4615b6d1d"), - ("web_bluetooth_remote_cddl", "web-bluetooth-remote-cddl.cddl", "25bef59834f25d7113e123898b952f6d37b78836b05c1a3b58012879ccebe176"), ("webdriver_bidi_all_cddl", "webdriver-bidi-all.cddl", "58baffeda459e2fee2d9b58a896eb4394e87e861ada5cb761a169d32e122138f"), - ("webdriver_bidi_local_cddl", "webdriver-bidi-local-cddl.cddl", "a88eec3567ed06b3a3884f9b4ef334c1726a64050e4290ae66c417ef1c77ea06"), - ("webdriver_bidi_remote_cddl", "webdriver-bidi-remote-cddl.cddl", "e51f3f0103664842b3c1211d4c6726b014658945d6166155054617904f40cbe0"), ] def webref_cddl(): diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 1eabb25d447c9..74b4e0c9c95e7 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -28,6 +28,14 @@ py_binary( ], ) +py_binary( + name = "update_cddl", + srcs = ["update_cddl.py"], + deps = [ + requirement("urllib3"), + ], +) + py_binary( name = "update_copyright", srcs = ["update_copyright.py"], diff --git a/scripts/update_cddl.py b/scripts/update_cddl.py new file mode 100644 index 0000000000000..d78b29d3a4a1a --- /dev/null +++ b/scripts/update_cddl.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +"""Update the pinned CDDL spec files downloaded from w3c/webref. + +The WebDriver BiDi (and related) CDDL grammars are not published as an npm +package; they are extracted from the edited specs and committed to the +``ed/cddl`` directory of https://github.com/w3c/webref . We pin a single +webref commit plus a sha256 for the ``-all`` (union) CDDL file of each +protocol in that directory (see ``common/webref_cddl.bzl``) so the Bazel +build fetches them reproducibly. The per-end ``-local``/``-remote`` splits are +skipped: generation merges the union, so only the ``-all`` files are consumed. + +This script repoints that pin at the tip of webref's "main" branch (the +continuous reffy extraction; the "curated" branch is a separate published +lineage with no shared history, so main keeps pin-to-pin diffs auditable), +refreshes every hash, and picks up files that upstream has added or removed. +Because the file set drifts over time, it regenerates both: + + - the ``_COMMIT`` and ``_CDDL_FILES`` entries in ``common/webref_cddl.bzl`` + - the matching ``use_repo(...)`` list for the extension in ``MODULE.bazel`` + +----------------------------------------------------------------------------- +usage: update_cddl.py [-h] [--commit COMMIT] [--branch BRANCH] + +options: + -h, --help show this help message and exit + --commit COMMIT pin this exact webref commit instead of the branch tip + --branch BRANCH webref branch to resolve when --commit is omitted (default: main) +----------------------------------------------------------------------------- +""" + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path + +import urllib3 + +http = urllib3.PoolManager() +root_dir = Path(os.path.realpath(__file__)).parent.parent + +REPO = "w3c/webref" +CDDL_PATH = "ed/cddl" +API_HEADERS = {"Accept": "application/vnd.github+json", "User-Agent": "selenium-update-cddl"} + +BZL_FILE = root_dir / "common" / "webref_cddl.bzl" +MODULE_FILE = root_dir / "MODULE.bazel" + + +def resolve_commit(branch): + r = http.request("GET", f"https://api.github.com/repos/{REPO}/commits/{branch}", headers=API_HEADERS) + if r.status != 200: + raise RuntimeError(f"Failed to resolve {REPO}@{branch}: HTTP {r.status}") + return json.loads(r.data)["sha"] + + +def list_cddl_files(commit): + r = http.request( + "GET", + f"https://api.github.com/repos/{REPO}/contents/{CDDL_PATH}?ref={commit}", + headers=API_HEADERS, + ) + if r.status != 200: + raise RuntimeError(f"Failed to list {CDDL_PATH} at {commit}: HTTP {r.status}") + entries = json.loads(r.data) + # Only the "-all" union of each protocol is consumed; the local/remote splits + # feed nothing (BiDi generation merges the union), so they are not pinned. + return sorted(e["name"] for e in entries if e["type"] == "file" and e["name"].endswith("-all.cddl")) + + +def repo_name(filename): + """Derive the Bazel repo name from a CDDL filename. + + ``at-driver-all.cddl`` -> ``at_driver_all_cddl`` + """ + return filename[: -len(".cddl")].replace("-", "_") + "_cddl" + + +def sha256_of(commit, filename): + url = f"https://raw.githubusercontent.com/{REPO}/{commit}/{CDDL_PATH}/{filename}" + r = http.request("GET", url) + if r.status != 200: + raise RuntimeError(f"Failed to download {url}: HTTP {r.status}") + return hashlib.sha256(r.data).hexdigest() + + +def build_entries(commit, filenames): + return [(repo_name(name), name, sha256_of(commit, name)) for name in filenames] + + +def existing_repo_names(content): + return set(re.findall(r'\(\s*"([a-z0-9_]+)"\s*,\s*"[^"]+\.cddl"', content)) + + +def render_cddl_files(entries): + lines = ["_CDDL_FILES = ["] + for name, filename, sha256 in entries: + lines.append(f' ("{name}", "{filename}", "{sha256}"),') + lines.append("]") + return "\n".join(lines) + + +def update_pin(commit, entries): + content = BZL_FILE.read_text() + before = existing_repo_names(content) + + content, n = re.subn(r'_COMMIT = "[0-9a-f]+"', f'_COMMIT = "{commit}"', content) + if n != 1: + raise RuntimeError(f"Expected exactly one _COMMIT assignment in {BZL_FILE.name}, found {n}") + content, n = re.subn(r"_CDDL_FILES = \[.*?\n\]", lambda _: render_cddl_files(entries), content, flags=re.S) + if n != 1: + raise RuntimeError(f"Expected exactly one _CDDL_FILES block in {BZL_FILE.name}, found {n}") + + BZL_FILE.write_text(content) + + after = {name for name, _, _ in entries} + return sorted(after - before), sorted(before - after) + + +def update_module(entries): + content = MODULE_FILE.read_text() + repo_list = "\n".join(f' "{name}",' for name, _, _ in sorted(entries)) + new_block = f"use_repo(\n webref_cddl_extension,\n{repo_list}\n)" + content, count = re.subn( + r"use_repo\(\n webref_cddl_extension,\n.*?\n\)", + new_block, + content, + flags=re.S, + ) + if count != 1: + raise RuntimeError(f"Expected exactly one webref_cddl_extension use_repo block, found {count}") + MODULE_FILE.write_text(content) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--commit", help="pin this exact webref commit instead of the branch tip") + parser.add_argument( + "--branch", + default="main", + help="webref branch to resolve when --commit is omitted (default: main)", + ) + args = parser.parse_args() + + commit = args.commit or resolve_commit(args.branch) + print(f"Pinning {REPO}@{commit}") + + filenames = list_cddl_files(commit) + print(f"Found {len(filenames)} CDDL files in {CDDL_PATH}") + + entries = build_entries(commit, filenames) + added, removed = update_pin(commit, entries) + update_module(entries) + + for name in added: + print(f" added: {name}") + for name in removed: + print(f" removed: {name}") + print(f"Updated {BZL_FILE.relative_to(root_dir)} and {MODULE_FILE.relative_to(root_dir)}") + + +if __name__ == "__main__": + main()