Carry secret-backed env vars from old CI into pyproject (lossless migration)

wads/data/skills/wads-migrate/SKILL.md

@@ -84,6 +84,16 @@ wads-migrate ci-to-stub --pin @v0.1.81
 `ci-to-uv` first so the per-repo `[tool.wads.ci]` audit happens before the
 inline workflow disappears.
 
+**Secrets are carried over automatically.** Both `ci-to-uv` and `ci-to-stub`
+(and `fleet-stub`) scan the *existing* workflow's `env:` blocks for
+`${{ secrets.X }}` references and inject any not-yet-declared ones into
+`[tool.wads.ci.env].extra_envvars` (recording a `secret_aliases` entry when the
+env-var name differs from the secret name). So a migration is **lossless** — you
+don't lose secrets that were wired only in the old YAML. The command prints what
+it carried; review them afterward and promote to `required`/`test` if a test
+truly depends on one. Use `wads-secrets add` only for secrets the old workflow
+did **not** already reference (e.g. a brand-new dependency).
+
 ## New Project
 
 ```bash

wads/fleet_migrate.py

@@ -21,7 +21,7 @@
 from pathlib import Path
 from typing import Iterable, Iterator, Optional
 
-from wads.migration import migrate_ci_to_stub
+from wads.migration import carry_ci_env_into_pyproject, migrate_ci_to_stub
 
 
 DEFAULT_STATE_FILE = Path.home() / "Downloads" / "wads_ci_diagnosis.json"
@@ -213,19 +213,44 @@ def migrate_one_to_stub(
     if not ci_path.exists():
         return RepoResult(name, repo_path, "skip", f"no {workflow_path}")
 
+    old_content = ci_path.read_text()
+
+    # Carry secret-backed env vars from the existing workflow into
+    # [tool.wads.ci.env] so the migration is lossless (and so the stub's
+    # transport list is generated to include them). Must run before
+    # migrate_ci_to_stub, which reads pyproject to render the secrets block.
+    carried: list = []
+    pyproject = Path(repo_path) / "pyproject.toml"
+    if pyproject.is_file():
+        try:
+            carried = carry_ci_env_into_pyproject(old_content, pyproject)
+        except Exception as e:  # never let env carry abort the batch
+            return RepoResult(name, repo_path, "fail", f"env-carry: {e}")
+
     try:
         new_content = migrate_ci_to_stub(str(ci_path), pin=pin)
     except Exception as e:
         return RepoResult(name, repo_path, "fail", f"ci-to-stub: {e}")
 
-    if ci_path.read_text() == new_content:
+    ci_changed = old_content != new_content
+    if not ci_changed and not carried:
         return RepoResult(name, repo_path, "noop", "already on this stub")
-    ci_path.write_text(new_content)
+    if ci_changed:
+        ci_path.write_text(new_content)
+
+    add_paths = [workflow_path] if ci_changed else []
+    if carried:
+        add_paths.append("pyproject.toml")
+
+    commit_detail = "switch to wads reusable workflow stub"
+    if carried:
+        commit_detail += f"; carry {len(carried)} env var(s) into [tool.wads.ci.env]"
+    msg = commit_message if not carried else f"ci: {commit_detail}"
 
     for cmd, label in (
-        (["git", "-C", repo_path, "add", workflow_path], "git add"),
+        (["git", "-C", repo_path, "add", *add_paths], "git add"),
         (
-            ["git", "-C", repo_path, "commit", "-m", commit_message, "--quiet"],
+            ["git", "-C", repo_path, "commit", "-m", msg, "--quiet"],
             "git commit",
         ),
         (
@@ -239,7 +264,10 @@ def migrate_one_to_stub(
                 name, repo_path, "fail", f"{label}: {r.stderr.strip()[:80]}"
             )
 
-    return RepoResult(name, repo_path, "ok", f"pushed to {default_branch}")
+    detail = f"pushed to {default_branch}"
+    if carried:
+        detail += f" (carried env: {', '.join(carried)})"
+    return RepoResult(name, repo_path, "ok", detail)
 
 
 def fleet_stub(

wads/migration.py

@@ -30,6 +30,7 @@
 """
 
 import os
+import re
 import sys
 from typing import Union, Mapping, Callable, Optional
 from pathlib import Path
@@ -936,6 +937,102 @@ def _find_pyproject_near(old_ci) -> Path | None:
     return None
 
 
+# Reference to a GitHub secret inside a workflow expression, e.g.
+# ``${{ secrets.OPENAI_API_KEY }}`` or ``${{ secrets.HF_WRITE_TOKEN || '' }}``.
+_SECRET_REF_RE = re.compile(r"\$\{\{\s*secrets\.([A-Za-z_][A-Za-z0-9_]*)")
+
+# Secrets the workflow handles structurally (publish auth, GitHub token); never
+# carried as repo env vars.
+_INFRA_SECRETS = frozenset({"GITHUB_TOKEN", "PYPI_PASSWORD", "TEST_PYPI_PASSWORD"})
+
+
+def _iter_env_blocks(node):
+    """Yield every ``env:`` mapping found anywhere in a parsed workflow tree."""
+    if isinstance(node, dict):
+        for key, value in node.items():
+            if key == "env" and isinstance(value, dict):
+                yield value
+            else:
+                yield from _iter_env_blocks(value)
+    elif isinstance(node, list):
+        for item in node:
+            yield from _iter_env_blocks(item)
+
+
+def extract_ci_env_vars(ci_text: str) -> dict:
+    """Extract secret-backed env vars from a CI workflow's ``env:`` blocks.
+
+    Returns a mapping ``{ENV_VAR: SECRET_NAME}`` for every workflow/job/step
+    ``env:`` entry whose value references ``${{ secrets.X }}``. This is the
+    per-repo signal of which secrets the tests actually consume.
+
+    Deliberately excluded:
+
+    * Literal env vars (``PROJECT_NAME``, ``LOG_LEVEL: DEBUG`` …) — no secret ref.
+    * Infra secrets (``GITHUB_TOKEN``, ``PYPI_PASSWORD``, ``TEST_PYPI_PASSWORD``).
+    * Reusable-workflow ``secrets:`` pass-through blocks — that's *transport*,
+      not a signal of usage, and scanning it would re-introduce the very
+      over-assignment the new model removes.
+
+    >>> ci = '''
+    ... env:
+    ...   PROJECT_NAME: myproj
+    ...   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || '' }}
+    ...   HF_TOKEN: ${{ secrets.HF_WRITE_TOKEN }}
+    ... jobs:
+    ...   publish:
+    ...     steps:
+    ...       - uses: x
+    ...         with:
+    ...           pypi-token: ${{ secrets.PYPI_PASSWORD }}
+    ... '''
+    >>> extract_ci_env_vars(ci) == {
+    ...     "OPENAI_API_KEY": "OPENAI_API_KEY", "HF_TOKEN": "HF_WRITE_TOKEN"}
+    True
+    """
+    import yaml
+
+    try:
+        data = yaml.safe_load(ci_text)
+    except yaml.YAMLError:
+        return {}
+    out: dict = {}
+    for env_map in _iter_env_blocks(data):
+        for var, value in env_map.items():
+            if not isinstance(value, str):
+                continue
+            match = _SECRET_REF_RE.search(value)
+            if match and str(var) not in _INFRA_SECRETS:
+                out[str(var)] = match.group(1)
+    return out
+
+
+def carry_ci_env_into_pyproject(
+    ci_text, pyproject_path, *, kind: str = "extra"
+) -> list:
+    """Merge secret-backed env vars from ``ci_text`` into ``[tool.wads.ci.env]``.
+
+    Adds only env vars not already declared (in any bucket), into the ``kind``
+    bucket (default ``extra`` — available-if-set, never fails the build), and
+    records a ``secret_aliases`` entry when the env var name differs from the
+    backing secret. Returns the list of newly-added var names (``[]`` if there
+    was nothing to carry). This is what makes ``ci-to-stub`` / ``ci-to-uv``
+    *lossless*: secrets wired only in the old workflow YAML are preserved in
+    ``pyproject.toml`` instead of silently dropped.
+    """
+    env_vars = extract_ci_env_vars(ci_text)
+    if not env_vars:
+        return []
+    from wads.secrets_cli import add_env_var_to_pyproject
+
+    added = []
+    for var, secret in env_vars.items():
+        changed, _ = add_env_var_to_pyproject(pyproject_path, var, secret, kind=kind)
+        if changed:
+            added.append(var)
+    return added
+
+
 def main():
     """CLI entry point for wads migration tools."""
     import argparse
@@ -1127,6 +1224,14 @@ def main():
                 print(f"Error: {input_path} not found", file=sys.stderr)
                 sys.exit(1)
 
+            # Carry secret-backed env vars from the OLD workflow into
+            # [tool.wads.ci.env] before re-rendering, so the new uv workflow's
+            # env block (generated from pyproject) preserves them.
+            pyproject = _find_pyproject_near(str(input_path))
+            carried = []
+            if pyproject is not None:
+                carried = carry_ci_env_into_pyproject(input_path.read_text(), pyproject)
+
             # Perform migration
             result = migrate_ci_to_uv(str(input_path))
 
@@ -1139,6 +1244,12 @@ def main():
             else:
                 print(result)
 
+            if carried:
+                print(
+                    f"✓ Carried {len(carried)} env var(s) into "
+                    f"[tool.wads.ci.env].extra_envvars: {', '.join(carried)}",
+                    file=sys.stderr,
+                )
             print(
                 "\nNote: Ensure your pyproject.toml has a [tool.wads.ci] section "
                 "and that secrets.PYPI_PASSWORD is a PyPI API token.",
@@ -1185,12 +1296,28 @@ def main():
                 )
                 sys.exit(2)
 
+            # Carry secret-backed env vars from the existing workflow into
+            # [tool.wads.ci.env] so the stub transports + exports them (lossless
+            # migration). Runs before stub generation, which reads pyproject.
+            pyproject = _find_pyproject_near(str(input_path))
+            carried = []
+            if pyproject is not None:
+                carried = carry_ci_env_into_pyproject(existing, pyproject)
+
             result = migrate_ci_to_stub(str(input_path), pin=args.pin)
 
             output_path = Path(args.output) if args.output else input_path
             output_path.parent.mkdir(parents=True, exist_ok=True)
             output_path.write_text(result)
             print(f"✓ Stub-ified {input_path} -> {output_path}")
+            if carried:
+                print(
+                    f"✓ Carried {len(carried)} env var(s) into "
+                    f"[tool.wads.ci.env].extra_envvars: {', '.join(carried)}\n"
+                    "  Review them (promote to required/test if appropriate) and "
+                    "ensure the matching GitHub secrets are set.",
+                    file=sys.stderr,
+                )
             if args.pin == "@master":
                 print(
                     "\nPinned to @master (floats with wads). If you need version "

wads/tests/test_ci_env_extraction.py

@@ -0,0 +1,94 @@
+"""Tests for lossless CI env extraction during migration (issue #45 follow-up).
+
+`wads-migrate ci-to-stub` / `ci-to-uv` / `fleet-stub` must carry secret-backed
+env vars declared only in the old workflow YAML into `[tool.wads.ci.env]`, so a
+migration never silently drops secrets the tests rely on.
+"""
+
+import pytest
+
+from wads.migration import carry_ci_env_into_pyproject, extract_ci_env_vars
+
+INLINE_UV_CI = """\
+name: Continuous Integration (uv)
+on: [push, pull_request]
+env:
+  PROJECT_NAME: myproj
+  LOG_LEVEL: DEBUG
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || '' }}
+  HF_TOKEN: ${{ secrets.HF_WRITE_TOKEN }}
+jobs:
+  validation:
+    steps:
+      - run: pytest
+  publish:
+    steps:
+      - uses: i2mint/wads/actions/pypi-publish-uv@master
+        with:
+          pypi-token: ${{ secrets.PYPI_PASSWORD }}
+      - uses: x
+        env:
+          KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
+"""
+
+STUB_CI = """\
+name: Continuous Integration
+on: [push, pull_request]
+jobs:
+  ci:
+    uses: i2mint/wads/.github/workflows/uv-ci.yml@master
+    secrets:
+      PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+"""
+
+
+def test_extract_from_env_blocks_only():
+    got = extract_ci_env_vars(INLINE_UV_CI)
+    # workflow-level + step-level env, alias resolved, literals/infra excluded
+    assert got == {
+        "OPENAI_API_KEY": "OPENAI_API_KEY",
+        "HF_TOKEN": "HF_WRITE_TOKEN",
+        "KAGGLE_KEY": "KAGGLE_KEY",
+    }
+    # PROJECT_NAME (literal) and PYPI_PASSWORD (infra, and in `with:` not `env:`)
+    assert "PROJECT_NAME" not in got and "PYPI_PASSWORD" not in got
+
+
+def test_stub_secrets_passthrough_is_ignored():
+    # A reusable-workflow `secrets:` block is transport, not env usage.
+    assert extract_ci_env_vars(STUB_CI) == {}
+
+
+def test_extract_handles_garbage():
+    assert extract_ci_env_vars(":: not yaml ::\n  - [") == {}
+    assert extract_ci_env_vars("") == {}
+
+
+@pytest.fixture
+def pyproject(tmp_path):
+    p = tmp_path / "pyproject.toml"
+    p.write_text(
+        '[project]\nname = "demo"\n\n'
+        "[tool.wads.ci.env]\n"
+        "required_envvars = []\ntest_envvars = []\nextra_envvars = []\ndefaults = {}\n"
+    )
+    return p
+
+
+def test_carry_adds_vars_and_alias(pyproject):
+    added = carry_ci_env_into_pyproject(INLINE_UV_CI, pyproject)
+    assert set(added) == {"OPENAI_API_KEY", "HF_TOKEN", "KAGGLE_KEY"}
+    text = pyproject.read_text()
+    assert "OPENAI_API_KEY" in text and "KAGGLE_KEY" in text
+    assert 'HF_TOKEN = "HF_WRITE_TOKEN"' in text  # alias recorded
+
+
+def test_carry_is_idempotent(pyproject):
+    carry_ci_env_into_pyproject(INLINE_UV_CI, pyproject)
+    again = carry_ci_env_into_pyproject(INLINE_UV_CI, pyproject)
+    assert again == []  # nothing new to add the second time
+
+
+def test_carry_noop_when_nothing_to_carry(pyproject):
+    assert carry_ci_env_into_pyproject(STUB_CI, pyproject) == []