diff --git a/wads/data/skills/wads-migrate/SKILL.md b/wads/data/skills/wads-migrate/SKILL.md index 4f7a25f..64b256c 100644 --- a/wads/data/skills/wads-migrate/SKILL.md +++ b/wads/data/skills/wads-migrate/SKILL.md @@ -84,6 +84,16 @@ wads-migrate ci-to-stub --pin @v0.1.81 `ci-to-uv` first so the per-repo `[tool.wads.ci]` audit happens before the inline workflow disappears. +**Secrets are carried over automatically.** Both `ci-to-uv` and `ci-to-stub` +(and `fleet-stub`) scan the *existing* workflow's `env:` blocks for +`${{ secrets.X }}` references and inject any not-yet-declared ones into +`[tool.wads.ci.env].extra_envvars` (recording a `secret_aliases` entry when the +env-var name differs from the secret name). So a migration is **lossless** — you +don't lose secrets that were wired only in the old YAML. The command prints what +it carried; review them afterward and promote to `required`/`test` if a test +truly depends on one. Use `wads-secrets add` only for secrets the old workflow +did **not** already reference (e.g. a brand-new dependency). + ## New Project ```bash diff --git a/wads/fleet_migrate.py b/wads/fleet_migrate.py index bf01b44..4051060 100644 --- a/wads/fleet_migrate.py +++ b/wads/fleet_migrate.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Iterable, Iterator, Optional -from wads.migration import migrate_ci_to_stub +from wads.migration import carry_ci_env_into_pyproject, migrate_ci_to_stub DEFAULT_STATE_FILE = Path.home() / "Downloads" / "wads_ci_diagnosis.json" @@ -213,19 +213,44 @@ def migrate_one_to_stub( if not ci_path.exists(): return RepoResult(name, repo_path, "skip", f"no {workflow_path}") + old_content = ci_path.read_text() + + # Carry secret-backed env vars from the existing workflow into + # [tool.wads.ci.env] so the migration is lossless (and so the stub's + # transport list is generated to include them). Must run before + # migrate_ci_to_stub, which reads pyproject to render the secrets block. + carried: list = [] + pyproject = Path(repo_path) / "pyproject.toml" + if pyproject.is_file(): + try: + carried = carry_ci_env_into_pyproject(old_content, pyproject) + except Exception as e: # never let env carry abort the batch + return RepoResult(name, repo_path, "fail", f"env-carry: {e}") + try: new_content = migrate_ci_to_stub(str(ci_path), pin=pin) except Exception as e: return RepoResult(name, repo_path, "fail", f"ci-to-stub: {e}") - if ci_path.read_text() == new_content: + ci_changed = old_content != new_content + if not ci_changed and not carried: return RepoResult(name, repo_path, "noop", "already on this stub") - ci_path.write_text(new_content) + if ci_changed: + ci_path.write_text(new_content) + + add_paths = [workflow_path] if ci_changed else [] + if carried: + add_paths.append("pyproject.toml") + + commit_detail = "switch to wads reusable workflow stub" + if carried: + commit_detail += f"; carry {len(carried)} env var(s) into [tool.wads.ci.env]" + msg = commit_message if not carried else f"ci: {commit_detail}" for cmd, label in ( - (["git", "-C", repo_path, "add", workflow_path], "git add"), + (["git", "-C", repo_path, "add", *add_paths], "git add"), ( - ["git", "-C", repo_path, "commit", "-m", commit_message, "--quiet"], + ["git", "-C", repo_path, "commit", "-m", msg, "--quiet"], "git commit", ), ( @@ -239,7 +264,10 @@ def migrate_one_to_stub( name, repo_path, "fail", f"{label}: {r.stderr.strip()[:80]}" ) - return RepoResult(name, repo_path, "ok", f"pushed to {default_branch}") + detail = f"pushed to {default_branch}" + if carried: + detail += f" (carried env: {', '.join(carried)})" + return RepoResult(name, repo_path, "ok", detail) def fleet_stub( diff --git a/wads/migration.py b/wads/migration.py index c429aa6..74e3c79 100644 --- a/wads/migration.py +++ b/wads/migration.py @@ -30,6 +30,7 @@ """ import os +import re import sys from typing import Union, Mapping, Callable, Optional from pathlib import Path @@ -936,6 +937,102 @@ def _find_pyproject_near(old_ci) -> Path | None: return None +# Reference to a GitHub secret inside a workflow expression, e.g. +# ``${{ secrets.OPENAI_API_KEY }}`` or ``${{ secrets.HF_WRITE_TOKEN || '' }}``. +_SECRET_REF_RE = re.compile(r"\$\{\{\s*secrets\.([A-Za-z_][A-Za-z0-9_]*)") + +# Secrets the workflow handles structurally (publish auth, GitHub token); never +# carried as repo env vars. +_INFRA_SECRETS = frozenset({"GITHUB_TOKEN", "PYPI_PASSWORD", "TEST_PYPI_PASSWORD"}) + + +def _iter_env_blocks(node): + """Yield every ``env:`` mapping found anywhere in a parsed workflow tree.""" + if isinstance(node, dict): + for key, value in node.items(): + if key == "env" and isinstance(value, dict): + yield value + else: + yield from _iter_env_blocks(value) + elif isinstance(node, list): + for item in node: + yield from _iter_env_blocks(item) + + +def extract_ci_env_vars(ci_text: str) -> dict: + """Extract secret-backed env vars from a CI workflow's ``env:`` blocks. + + Returns a mapping ``{ENV_VAR: SECRET_NAME}`` for every workflow/job/step + ``env:`` entry whose value references ``${{ secrets.X }}``. This is the + per-repo signal of which secrets the tests actually consume. + + Deliberately excluded: + + * Literal env vars (``PROJECT_NAME``, ``LOG_LEVEL: DEBUG`` …) — no secret ref. + * Infra secrets (``GITHUB_TOKEN``, ``PYPI_PASSWORD``, ``TEST_PYPI_PASSWORD``). + * Reusable-workflow ``secrets:`` pass-through blocks — that's *transport*, + not a signal of usage, and scanning it would re-introduce the very + over-assignment the new model removes. + + >>> ci = ''' + ... env: + ... PROJECT_NAME: myproj + ... OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || '' }} + ... HF_TOKEN: ${{ secrets.HF_WRITE_TOKEN }} + ... jobs: + ... publish: + ... steps: + ... - uses: x + ... with: + ... pypi-token: ${{ secrets.PYPI_PASSWORD }} + ... ''' + >>> extract_ci_env_vars(ci) == { + ... "OPENAI_API_KEY": "OPENAI_API_KEY", "HF_TOKEN": "HF_WRITE_TOKEN"} + True + """ + import yaml + + try: + data = yaml.safe_load(ci_text) + except yaml.YAMLError: + return {} + out: dict = {} + for env_map in _iter_env_blocks(data): + for var, value in env_map.items(): + if not isinstance(value, str): + continue + match = _SECRET_REF_RE.search(value) + if match and str(var) not in _INFRA_SECRETS: + out[str(var)] = match.group(1) + return out + + +def carry_ci_env_into_pyproject( + ci_text, pyproject_path, *, kind: str = "extra" +) -> list: + """Merge secret-backed env vars from ``ci_text`` into ``[tool.wads.ci.env]``. + + Adds only env vars not already declared (in any bucket), into the ``kind`` + bucket (default ``extra`` — available-if-set, never fails the build), and + records a ``secret_aliases`` entry when the env var name differs from the + backing secret. Returns the list of newly-added var names (``[]`` if there + was nothing to carry). This is what makes ``ci-to-stub`` / ``ci-to-uv`` + *lossless*: secrets wired only in the old workflow YAML are preserved in + ``pyproject.toml`` instead of silently dropped. + """ + env_vars = extract_ci_env_vars(ci_text) + if not env_vars: + return [] + from wads.secrets_cli import add_env_var_to_pyproject + + added = [] + for var, secret in env_vars.items(): + changed, _ = add_env_var_to_pyproject(pyproject_path, var, secret, kind=kind) + if changed: + added.append(var) + return added + + def main(): """CLI entry point for wads migration tools.""" import argparse @@ -1127,6 +1224,14 @@ def main(): print(f"Error: {input_path} not found", file=sys.stderr) sys.exit(1) + # Carry secret-backed env vars from the OLD workflow into + # [tool.wads.ci.env] before re-rendering, so the new uv workflow's + # env block (generated from pyproject) preserves them. + pyproject = _find_pyproject_near(str(input_path)) + carried = [] + if pyproject is not None: + carried = carry_ci_env_into_pyproject(input_path.read_text(), pyproject) + # Perform migration result = migrate_ci_to_uv(str(input_path)) @@ -1139,6 +1244,12 @@ def main(): else: print(result) + if carried: + print( + f"✓ Carried {len(carried)} env var(s) into " + f"[tool.wads.ci.env].extra_envvars: {', '.join(carried)}", + file=sys.stderr, + ) print( "\nNote: Ensure your pyproject.toml has a [tool.wads.ci] section " "and that secrets.PYPI_PASSWORD is a PyPI API token.", @@ -1185,12 +1296,28 @@ def main(): ) sys.exit(2) + # Carry secret-backed env vars from the existing workflow into + # [tool.wads.ci.env] so the stub transports + exports them (lossless + # migration). Runs before stub generation, which reads pyproject. + pyproject = _find_pyproject_near(str(input_path)) + carried = [] + if pyproject is not None: + carried = carry_ci_env_into_pyproject(existing, pyproject) + result = migrate_ci_to_stub(str(input_path), pin=args.pin) output_path = Path(args.output) if args.output else input_path output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(result) print(f"✓ Stub-ified {input_path} -> {output_path}") + if carried: + print( + f"✓ Carried {len(carried)} env var(s) into " + f"[tool.wads.ci.env].extra_envvars: {', '.join(carried)}\n" + " Review them (promote to required/test if appropriate) and " + "ensure the matching GitHub secrets are set.", + file=sys.stderr, + ) if args.pin == "@master": print( "\nPinned to @master (floats with wads). If you need version " diff --git a/wads/tests/test_ci_env_extraction.py b/wads/tests/test_ci_env_extraction.py new file mode 100644 index 0000000..472c225 --- /dev/null +++ b/wads/tests/test_ci_env_extraction.py @@ -0,0 +1,94 @@ +"""Tests for lossless CI env extraction during migration (issue #45 follow-up). + +`wads-migrate ci-to-stub` / `ci-to-uv` / `fleet-stub` must carry secret-backed +env vars declared only in the old workflow YAML into `[tool.wads.ci.env]`, so a +migration never silently drops secrets the tests rely on. +""" + +import pytest + +from wads.migration import carry_ci_env_into_pyproject, extract_ci_env_vars + +INLINE_UV_CI = """\ +name: Continuous Integration (uv) +on: [push, pull_request] +env: + PROJECT_NAME: myproj + LOG_LEVEL: DEBUG + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || '' }} + HF_TOKEN: ${{ secrets.HF_WRITE_TOKEN }} +jobs: + validation: + steps: + - run: pytest + publish: + steps: + - uses: i2mint/wads/actions/pypi-publish-uv@master + with: + pypi-token: ${{ secrets.PYPI_PASSWORD }} + - uses: x + env: + KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} +""" + +STUB_CI = """\ +name: Continuous Integration +on: [push, pull_request] +jobs: + ci: + uses: i2mint/wads/.github/workflows/uv-ci.yml@master + secrets: + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +""" + + +def test_extract_from_env_blocks_only(): + got = extract_ci_env_vars(INLINE_UV_CI) + # workflow-level + step-level env, alias resolved, literals/infra excluded + assert got == { + "OPENAI_API_KEY": "OPENAI_API_KEY", + "HF_TOKEN": "HF_WRITE_TOKEN", + "KAGGLE_KEY": "KAGGLE_KEY", + } + # PROJECT_NAME (literal) and PYPI_PASSWORD (infra, and in `with:` not `env:`) + assert "PROJECT_NAME" not in got and "PYPI_PASSWORD" not in got + + +def test_stub_secrets_passthrough_is_ignored(): + # A reusable-workflow `secrets:` block is transport, not env usage. + assert extract_ci_env_vars(STUB_CI) == {} + + +def test_extract_handles_garbage(): + assert extract_ci_env_vars(":: not yaml ::\n - [") == {} + assert extract_ci_env_vars("") == {} + + +@pytest.fixture +def pyproject(tmp_path): + p = tmp_path / "pyproject.toml" + p.write_text( + '[project]\nname = "demo"\n\n' + "[tool.wads.ci.env]\n" + "required_envvars = []\ntest_envvars = []\nextra_envvars = []\ndefaults = {}\n" + ) + return p + + +def test_carry_adds_vars_and_alias(pyproject): + added = carry_ci_env_into_pyproject(INLINE_UV_CI, pyproject) + assert set(added) == {"OPENAI_API_KEY", "HF_TOKEN", "KAGGLE_KEY"} + text = pyproject.read_text() + assert "OPENAI_API_KEY" in text and "KAGGLE_KEY" in text + assert 'HF_TOKEN = "HF_WRITE_TOKEN"' in text # alias recorded + + +def test_carry_is_idempotent(pyproject): + carry_ci_env_into_pyproject(INLINE_UV_CI, pyproject) + again = carry_ci_env_into_pyproject(INLINE_UV_CI, pyproject) + assert again == [] # nothing new to add the second time + + +def test_carry_noop_when_nothing_to_carry(pyproject): + assert carry_ci_env_into_pyproject(STUB_CI, pyproject) == []