From 00bc81ee622aee7fb1bcbe6d0bd0496ea89de9e0 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 15:10:56 -0700 Subject: [PATCH 001/128] feat: add verifiable global sync foundation --- .gitignore | 1 + .pdd/repository-id | 1 + docs/global_sync_resolution_plan.md | 1657 +++++++++++++++++ pdd/commands/__init__.py | 5 + pdd/commands/sync_core.py | 328 ++++ pdd/core/cli.py | 6 + pdd/sync_core/__init__.py | 235 +++ pdd/sync_core/capabilities.py | 127 ++ pdd/sync_core/certificate.py | 825 ++++++++ pdd/sync_core/certificate_verifier.py | 97 + pdd/sync_core/classifier.py | 173 ++ pdd/sync_core/durability.py | 13 + pdd/sync_core/evidence_store.py | 204 ++ pdd/sync_core/finalize.py | 314 ++++ pdd/sync_core/fingerprint_store.py | 291 +++ pdd/sync_core/git_io.py | 15 + pdd/sync_core/identity.py | 97 + pdd/sync_core/includes.py | 370 ++++ pdd/sync_core/language.py | 182 ++ pdd/sync_core/lifecycle.py | 159 ++ pdd/sync_core/manifest.py | 929 +++++++++ pdd/sync_core/path_policy.py | 118 ++ pdd/sync_core/planner.py | 134 ++ pdd/sync_core/pytest_probe.py | 27 + pdd/sync_core/released_checker.py | 103 + pdd/sync_core/reporting.py | 358 ++++ pdd/sync_core/runner.py | 688 +++++++ pdd/sync_core/scenario_contract.py | 17 + pdd/sync_core/scenario_harness.py | 650 +++++++ pdd/sync_core/snapshot.py | 68 + pdd/sync_core/transaction.py | 510 +++++ pdd/sync_core/trust.py | 433 +++++ pdd/sync_core/types.py | 330 ++++ pdd/sync_core/verification.py | 246 +++ pdd/sync_core/waivers.py | 133 ++ pyproject.toml | 5 +- tests/test_sync_core_certificate.py | 481 +++++ tests/test_sync_core_certificate_verifier.py | 63 + tests/test_sync_core_classifier.py | 321 ++++ tests/test_sync_core_cli.py | 43 + tests/test_sync_core_evidence_store.py | 101 + tests/test_sync_core_fingerprint_store.py | 168 ++ tests/test_sync_core_identity_path_policy.py | 114 ++ tests/test_sync_core_includes.py | 249 +++ tests/test_sync_core_language.py | 45 + tests/test_sync_core_lifecycle_scenarios.py | 140 ++ tests/test_sync_core_manifest.py | 437 +++++ tests/test_sync_core_planner_capabilities.py | 107 ++ tests/test_sync_core_released_checker.py | 94 + tests/test_sync_core_reporting.py | 548 ++++++ tests/test_sync_core_runner.py | 229 +++ tests/test_sync_core_snapshot.py | 105 ++ tests/test_sync_core_transaction.py | 181 ++ tests/test_sync_core_trust.py | 175 ++ tests/test_sync_core_verification_profiles.py | 161 ++ 55 files changed, 13610 insertions(+), 1 deletion(-) create mode 100644 .pdd/repository-id create mode 100644 docs/global_sync_resolution_plan.md create mode 100644 pdd/commands/sync_core.py create mode 100644 pdd/sync_core/__init__.py create mode 100644 pdd/sync_core/capabilities.py create mode 100644 pdd/sync_core/certificate.py create mode 100644 pdd/sync_core/certificate_verifier.py create mode 100644 pdd/sync_core/classifier.py create mode 100644 pdd/sync_core/durability.py create mode 100644 pdd/sync_core/evidence_store.py create mode 100644 pdd/sync_core/finalize.py create mode 100644 pdd/sync_core/fingerprint_store.py create mode 100644 pdd/sync_core/git_io.py create mode 100644 pdd/sync_core/identity.py create mode 100644 pdd/sync_core/includes.py create mode 100644 pdd/sync_core/language.py create mode 100644 pdd/sync_core/lifecycle.py create mode 100644 pdd/sync_core/manifest.py create mode 100644 pdd/sync_core/path_policy.py create mode 100644 pdd/sync_core/planner.py create mode 100644 pdd/sync_core/pytest_probe.py create mode 100644 pdd/sync_core/released_checker.py create mode 100644 pdd/sync_core/reporting.py create mode 100644 pdd/sync_core/runner.py create mode 100644 pdd/sync_core/scenario_contract.py create mode 100644 pdd/sync_core/scenario_harness.py create mode 100644 pdd/sync_core/snapshot.py create mode 100644 pdd/sync_core/transaction.py create mode 100644 pdd/sync_core/trust.py create mode 100644 pdd/sync_core/types.py create mode 100644 pdd/sync_core/verification.py create mode 100644 pdd/sync_core/waivers.py create mode 100644 tests/test_sync_core_certificate.py create mode 100644 tests/test_sync_core_certificate_verifier.py create mode 100644 tests/test_sync_core_classifier.py create mode 100644 tests/test_sync_core_cli.py create mode 100644 tests/test_sync_core_evidence_store.py create mode 100644 tests/test_sync_core_fingerprint_store.py create mode 100644 tests/test_sync_core_identity_path_policy.py create mode 100644 tests/test_sync_core_includes.py create mode 100644 tests/test_sync_core_language.py create mode 100644 tests/test_sync_core_lifecycle_scenarios.py create mode 100644 tests/test_sync_core_manifest.py create mode 100644 tests/test_sync_core_planner_capabilities.py create mode 100644 tests/test_sync_core_released_checker.py create mode 100644 tests/test_sync_core_reporting.py create mode 100644 tests/test_sync_core_runner.py create mode 100644 tests/test_sync_core_snapshot.py create mode 100644 tests/test_sync_core_transaction.py create mode 100644 tests/test_sync_core_trust.py create mode 100644 tests/test_sync_core_verification_profiles.py diff --git a/.gitignore b/.gitignore index 003e63fe7..6d38ee8c5 100644 --- a/.gitignore +++ b/.gitignore @@ -242,6 +242,7 @@ yarn.lock # PDD internal directory (backups, locks, worktrees, etc.) .pdd/* !.pdd/ +!.pdd/repository-id !.pdd/meta/ .pdd/meta/*.json !.pdd/meta/agentic_bug_orchestrator_python.json diff --git a/.pdd/repository-id b/.pdd/repository-id new file mode 100644 index 000000000..016d40b53 --- /dev/null +++ b/.pdd/repository-id @@ -0,0 +1 @@ +3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0 diff --git a/docs/global_sync_resolution_plan.md b/docs/global_sync_resolution_plan.md new file mode 100644 index 000000000..533821514 --- /dev/null +++ b/docs/global_sync_resolution_plan.md @@ -0,0 +1,1657 @@ +# Global Sync Resolution Plan + +Status: implementation in progress; acceptance gates remain red +Last updated: 2026-07-10 +Tracking epic: [promptdriven/pdd#1932](https://github.com/promptdriven/pdd/issues/1932) +Primary consumer: `promptdriven/pdd_cloud` + +## 1. Executive decision + +The global sync guarantee is not complete. The merged implementation in PR #1954 +adds useful deterministic reporting and protects some conflict cases, but it does +not establish the invariant that prompts, included documentation, code, examples, +tests, architecture metadata, and fingerprints remain mutually consistent across +all edit channels. + +The program must be completed as a sequence of foundations, repair semantics, and +enforcement. The order is mandatory: + +1. Define one unit identity and dependency model. +2. Define one pure drift classifier. +3. Make artifact and fingerprint finalization transactional. +4. Route every command and automation path through those primitives. +5. Implement non-destructive repair and conflict resolution. +6. Enforce the invariant at PR, merge, and nightly boundaries. +7. Backfill consumer repositories only after the upstream implementation passes + lifecycle end-to-end tests. + +No bulk baseline, hook, or CI stamper may silently declare an arbitrary working +tree synchronized. A baseline can say what bytes were observed; it cannot, by +itself, prove that those bytes express the same intent. + +## 2. Evidence and current state + +This plan is based on an audit of `origin/main` at `c255f3bf` and the open global +sync branches as of 2026-07-09. + +- `origin/main` tracks 42 non-run-report fingerprint files. The in-progress + inventory in PR #1969 identifies 223 stampable units, leaving 181 intended + units without tracked fingerprints on `main`. +- A clean-checkout `pdd reconcile` on `origin/main` classified those 42 visible + units as 9 current, 7 stale, 20 conflicts, and 6 failures. +- Once any metadata exists, the merged discovery path defaults to metadata-backed + units. It therefore reports `unbaselined=0` while omitting prompt-backed units + that have no fingerprint. +- The issue #1932 test suite passed 13 tests before PR #1954 merged, but it drives + reporting-only forms of `sync`, `update`, `change`, and CI heal through the same + `build_report` helper. Its heal assertion re-stamps changed bytes; it does not + verify semantic propagation. +- Live mutation still relies on `sync_determine_operation`, command-specific + wrappers, and CI-only reclassification. +- There are ten direct `save_fingerprint` call sites in the open integration + branch. The base writer performs a direct write and reports persistence failure + as a warning. +- PR #1969 is blocked. Its required unit and auto-heal checks fail. A local run of + the merged issue #1932 acceptance suite against its head produced 9 failures and + 4 passes. +- PR #1937 sketches transactional finalization in prompt files but does not yet + contain the complete generated implementation and invariant test harness. +- `pdd_cloud` PR #3013 merged a deterministic prompt-side nightly reconciler, but + pdd_cloud issue #2994 remains open for code-side coverage and cleanup. The + historical consumer baseline was 377 stale fingerprints out of 788. + +These facts make PR #1954 a partial delivery, not completion of issue #1932. + +### 2.1 Execution ledger (2026-07-10) + +This ledger records implementation progress; it does not replace the Definition +of Done or authorize a partial green certificate. + +| Workstream | Current evidence | State | +| --- | --- | --- | +| Canonical core | Identity, immutable Git manifest, include closure, hashing, classifier, profiles, trust, reporting, transaction WAL, and compatibility routing implemented under `pdd/sync_core/` | implemented, not rolled out | +| Exact certification | Independent exact-SHA clones, clean-tree/ancestry checks, strict fresh green certificate verification, exact issuer and repository refs | implemented and unit tested | +| Trusted test runner | Strict pytest invocation; ambient options/plugins disabled; config, `conftest.py`, and transitive local helper closure signed; protected and candidate node IDs compared before each protected node executes independently | implemented for pytest only; non-pytest adapters remain | +| Temporal proof | Signed rows require complete predicates, repository identity, SHA ancestry, consecutive UTC dates, and no-write lifecycle metrics | implemented; seven real nights absent | +| Lifecycle harness | A credential-free checker-owned packaged harness replaces candidate pytest files; all required scenario IDs execute from an isolated built wheel | implemented locally; protected release/workflow deployment remains | +| PDD verification | `278 passed` for `tests/test_sync_core_*.py`; checker-owned scenarios also execute from an isolated built wheel | component green, E2E red | +| pdd_cloud boundary | Exact-tree path/Python/prompt/architecture scan reports zero forbidden vendored semantics at `677a9c88f`; the legacy finalizer declarations and implementation are removed and current consumer installs are exact-version pinned | implemented; final pin must target the protected reviewed release | +| pdd_cloud inventory | Exact `HEAD^..HEAD` report at `677a9c88f`: 572 current managed, 573 protected expected, 534 unbaselined/unknown, 38 drifted/failed, one removed-unit invalidity, zero profiles/evidence, and zero unaccounted tracked paths | two-phase decommission plus profiles/evidence migration remain | +| Global certificate | Scan remains red for protected checker deployment, transactional staging, profile/evidence debt, and nightly history | correctly blocked | + +No acceptance claim is valid until the signed certificate recomputes `ok: true` +for exact protected PDD and pdd_cloud SHAs and a verifier supplied with the +protected issuer, expected repository identities/SHAs, and current time accepts +it without candidate-controlled inputs. + +### 2.2 Critical unblock sequence + +The final certificate cannot be unblocked by weakening its predicate. Work must +proceed in this dependency order, with each exit check retained as evidence for +the next stage. + +| Order | Unblock | Required exit evidence | +| --- | --- | --- | +| 1 | Split the current dirty implementation into reviewable foundation, transaction, runner/trust, command-routing, and rollout commits. Do not activate enforcement in these commits. | Each commit has focused tests; the branch is clean; source and built-wheel suites agree. | +| 2 | Publish the checker from a protected reviewed tag and pin its wheel digest in protected CI. The released checker, not candidate code, owns scenario definitions, lifecycle metrics, certificate predicate recomputation, and signing. Candidate generators and tests run only in credential-free children. | A candidate change that returns unconditional PASS, edits lifecycle scenarios, or prints environment variables still produces a red unsigned observation; the released checker artifact digest and workflow identity are present in the signed certificate. | +| 3 | Replace direct production writes with generate-to-staging followed by one durable transaction containing artifacts, shared metadata, evidence, report, and fingerprint. Use descriptor-relative no-follow commit operations and verify all WAL blobs before the first destination change. | Process death at every generation/journal/install phase recovers to the complete old or complete new state; immediate rerun writes zero files. | +| 4 | Make the pytest adapter compare protected expected node IDs with collected and executed node IDs, and bind every config-loaded plugin and executable support module. Add the remaining declared adapters before claiming cross-language coverage. | Removed parametrized cases, config-loaded local plugins, collection hooks, skips, xfails, deselection, and dirty support all fail closed. | +| 5 | Route runtime preprocessing, dependency discovery, change detection, and legacy operation selection through the canonical include graph, identity, classifier, and transaction APIs. | No production module independently parses includes, hashes fingerprints, classifies drift, or writes canonical metadata; compatibility modules contain orchestration only. | +| 6 | Land and release the PDD foundation. Run the released checker against clean exact-SHA PDD clones, then create protected verification profiles and trusted evidence for every expected managed unit. | PDD exact-SHA report has complete inventory, 100% profiles/evidence, and zero red semantic/baseline states or waivers. | +| 7 | Pin that release in pdd_cloud, remove `metadata_finalize.py` and all other vendored sync semantics, resolve the duplicate output conflict, then migrate profiles, fingerprints, and evidence by bounded subtree PRs. | Independent semantic ownership audit and exact-tree scan both report zero consumer-owned sync semantics; pdd_cloud has 100% profiles/evidence and no red units or waivers. | +| 8 | Enable the protected merge-group lifecycle lane using clean clones and the pinned checker. Run every required injected scenario, including the real pdd_cloud canary, without required skips. | One fresh signed scan certificate has `scan_ok: true`, lifecycle failures/skips/errors/timeouts zero, and both no-write counters zero. | +| 9 | Start the temporal gate only after steps 1-8 are stable. Store signed immutable nightly rows outside either candidate checkout and run complete scans even after ledger/cursor deletion. | Seven consecutive UTC-date certificates preserve repository identity/SHA lineage; normal nights are no-op and injected drift is detected, resolved or blocked, then rerun with zero writes. | +| 10 | Run the documented final command and a fresh xhigh adversarial review. | Command exits 0; an independent verifier accepts the signed exact-SHA certificate; reviewer returns `APPROVE`. | + +Steps 1-7 are engineering work. Step 8 requires protected CI/release configuration. +Step 9 additionally requires seven elapsed nightly windows and therefore cannot +be replaced with locally fabricated timestamps. The current implementation is +at steps 1-4: released-wheel identity, exact pytest node collection proof, and +the checker-owned harness are implemented and built-wheel tested locally. +Protected release/workflow deployment, transactional generation-to-staging, +non-pytest adapters, and repository profile/evidence rollout are not complete. + +## 3. Scope + +### 3.1 In scope + +- Prompt, transitive included-document, code, example, and test synchronization. +- Architecture and `.pddrc` identity/path consistency. +- Fingerprint and run-report trustworthiness. +- Manual edits, coding-agent edits, PDD command edits, CI auto-heal edits, and + hotfix PRs. +- Nested projects, Git worktrees, duplicate leaf names, path-qualified modules, + missing files, renames, and deletions. +- Python and every language currently supported by PDD path resolution. +- Human-owned tests and artifacts that PDD validates but must not overwrite. +- Source checkout and installed-wheel behavior. +- The PDD repository as the first dogfood target and pdd_cloud as the first large + consumer rollout. + +### 3.2 Out of scope + +- A real-time background regeneration daemon. Commit, PR, merge, and nightly + boundaries are sufficient. +- PRD synchronization. It needs a separate semantic model after the core artifact + loop is trustworthy. +- Automatic acceptance of ambiguous conflicts. +- Proving arbitrary prose and arbitrary code are semantically equivalent with no + declared contract or executable evidence. Such units must report `UNKNOWN`, not + `IN_SYNC`. + +## 4. Definitions and invariants + +### 4.1 Three independent status dimensions + +The implementation must stop using one `IN_SYNC` label for three different +claims. + +| Dimension | Values | Meaning | +| --- | --- | --- | +| Inventory | `MANAGED`, `HUMAN_OWNED`, `REMOVED`, `INVALID` | Intrinsic unit/artifact ownership; waivers are a separate exception dimension | +| Baseline | `CURRENT`, `DRIFTED`, `UNBASELINED`, `CORRUPT` | Whether current bytes match a trustworthy fingerprint | +| Semantic | `VERIFIED`, `UNKNOWN`, `CONFLICT`, `FAILED` | Whether prompt-side intent and derived behavior have current evidence of agreement | + +`IN_SYNC` is a derived convenience verdict and is legal only when: + +```text +inventory == MANAGED +and baseline == CURRENT +and semantic == VERIFIED +and no required artifact is missing +and all required validation evidence is tied to the current artifact hashes +and every obligation in the unit's verification profile is trusted-verified +``` + +An explicit baseline reset produces `CURRENT + UNKNOWN`, not `IN_SYNC`. + +### 4.2 Required invariants + +1. **Complete inventory:** the candidate universe is the union of the base and head + manifests, configured source/test roots, prompt declarations, architecture + declarations, fingerprints, ownership records, and unmatched tracked artifacts. + Every candidate is classified as managed, human-owned, removed, or invalid, + with waiver state reported orthogonally. No unit disappears because its prompt, + declaration, and metadata were deleted together. +2. **One identity:** every layer resolves the same stable repository-relative unit + ID, checkout/config roots, artifact paths, and ownership. +3. **One classifier:** given the same snapshot and optional Git change set, every + consumer emits the same baseline and semantic statuses. +4. **Transactional success:** a mutating command cannot report success unless its + artifact writes, validation evidence, run report, and fingerprint commit as one + successful state transition. +5. **No silent acceptance:** no unattended command changes a baseline merely to + make drift disappear. +6. **No data loss:** a prompt-side and derived-side co-edit never auto-selects a + winner or deletes ancestor metadata. +7. **No silent skip:** every ignored, waived, unbaselined, invalid, failed, or + conflicted condition appears in human and JSON summaries, including gross + inventory counts before waivers. +8. **Idempotency:** after a successful repair, the same command run again performs + no writes and recommends no additional repair. +9. **Evidence freshness and trust:** tests, examples, surface checks, and run reports + count only when a trusted validator recomputes them or verifies a trusted + attestation bound to the complete input closure, manifest/config digest, runner + command/config, tool versions, result, base SHA, and checked head/merge-group SHA. + PR-authored status fields and evidence files cannot certify themselves. +10. **Boundary enforcement:** protected `main` accepts a managed unit only when it + satisfies the complete `IN_SYNC` predicate. `DRIFTED`, `UNBASELINED`, `CORRUPT`, + `UNKNOWN`, `CONFLICT`, `FAILED`, and `INVALID` block unless a trusted, + snapshot-bound, reviewable, expiring waiver applies. + +## 5. Target architecture + +### 5.1 Package boundaries + +Create a small sync core and migrate existing modules behind compatibility shims: + +```text +pdd/sync_core/ + types.py immutable IDs, snapshots, statuses, decisions + identity.py canonical project/unit/path/ownership resolution + path_policy.py protected read/write roots and no-follow validation + includes.py canonical include parser and dependency closure + hashing.py canonical artifact and composite prompt hashing + fingerprint_store.py schema migration, atomic read/write, locks + classifier.py pure drift and conflict classification + planner.py status -> allowed repair plans + capabilities.py command/agent/server mutation scopes and transaction policy + transaction.py artifact/fingerprint/evidence finalization + ledger.py durable changed-unit queue and acknowledgements + reporting.py stable human and JSON schema +``` + +Existing public APIs remain during migration: + +- `sync_determine_operation` becomes a compatibility adapter that adds workflow + progression and operation selection to the core classifier result. +- `operation_log.save_fingerprint` becomes a deprecated adapter around the + transaction/store API and is eventually internal-only. +- `continuous_sync` becomes command composition and reporting, not a second unit + resolver, hasher, or classifier. +- `ci_drift_heal`, `agentic_sync`, `update`, `change`, and global `sync` consume + the same core snapshot and decision types. + +### 5.2 Canonical unit identity + +Use a typed candidate identity for every artifact and derive a `UnitId` only after +prompt-backed adoption. Do not use a flattened metadata filename: + +```python +@dataclass(frozen=True) +class CandidateId: + repository_id: str + artifact_relpath: PurePosixPath + role: str + +@dataclass(frozen=True) +class UnitId: + repository_id: str + prompt_relpath: PurePosixPath + language_id: str + +SubjectId = UnitId | CandidateId + +@dataclass(frozen=True) +class ResolvedUnit: + unit_id: UnitId + checkout_root: Path + config_root: Path + basename: str + prompt_path: Path + code_path: Path | None + example_paths: tuple[Path, ...] + test_paths: tuple[Path, ...] + included_docs: tuple[Path, ...] + dependencies: tuple[UnitId, ...] + owned_artifacts: frozenset[str] + validated_artifacts: frozenset[str] + ownership_provenance: Mapping[str, str] + config_path: Path | None + architecture_path: Path | None + manifest_digest: str +``` + +Resolution precedence is explicit and tested: + +1. Read a stable UUID from the protected `.pdd/repository-id`, separately from the + checkout/worktree root and nearest governing `.pddrc`. A new repository + initializes it once in a reviewed setup commit. Fork PRs use the target base + repository ID; remote rename/transfer does not change it. An independent fork + requires an explicit protected rekey migration. Offline operation reads the + file; a legacy repository without one may report using an ephemeral ID but may + not write canonical v2 state until initialization. +2. Identify units by repository-relative POSIX prompt path plus stable language ID. + Absolute paths never enter persistent identity; basename and language spellings + are aliases, not identity. +3. Apply `.pddrc` path templates relative to the governing config directory. +4. Apply an exact architecture entry override only when it maps to the same prompt + identity. +5. Reject ambiguous leaf aliases and duplicate prompt-to-artifact mappings. +6. Include normalized role-to-path mappings, `.pddrc`, architecture, ownership, + graph declarations, and policy inputs in the manifest digest. A path-policy + retarget is drift even when the old and new files have identical bytes. +7. Compare the base and head manifests. A rename requires an explicit identity + migration; a removal requires a reviewed tombstone that accounts for every + previously owned artifact and fingerprint. +8. Compile an in-memory `UnitManifest`; all downstream consumers use it rather + than resolving paths again. + +Promptless code/tests and other unmatched artifacts retain `CandidateId` and can +be reported, owned, waived, removed, or later adopted into a `UnitId`. Adoption is +an explicit protected mapping; it cannot happen by basename guess. + +Reports, ledgers, waivers, tombstones, ownership transitions, and journals use a +discriminated `SubjectId`, never assume every candidate already has a `UnitId`. + +The manifest is deterministic output, not another hand-maintained source of +truth. An optional committed inventory lock is only a generated assertion used to +detect unexplained additions/removals. Tests compare manifests across source +checkout, wheel installation, root invocation, nested invocation, and Git +worktrees. The migration must adapt or retire the existing `ResolvedSyncUnit` +resolver so it cannot remain a parallel identity implementation. + +Repository-ID initialization, rekey, ownership transition, unit rename, adoption, +and removal are protected policy transitions evaluated against the base manifest. +Tests cover forks, renamed/transferred remotes, no-remotes/offline operation, +independent fork rekey, and legacy repositories. + +The resolver also emits a protected `expected_managed` assertion derived from base +and historical prompt, architecture, fingerprint, ownership, and generation +provenance. It fixes the sync-capable denominator independently of current debt: + +- A currently/previously generated or prompt-backed expected-managed subject cannot + move to `HUMAN_OWNED` while non-`IN_SYNC`. +- Permanent decommissioning is a separate protected PR starting from a trusted + `IN_SYNC` snapshot, contains no unrelated behavior change, accounts for every + artifact through a tombstone, and records rationale/owner. It is reported as + decommissioning, not drift burn-down. +- Activation and epic closure require zero managed-to-human transitions used to + discharge debt. Reports show expected-managed total, current managed total, + historical human-owned total, and all transitions. + +#### Protected path policy + +One protected-base `PathPolicy` governs every prompt, generated artifact, example, +test, architecture/config file, metadata file, runner config, evidence record, +staging file, journal, and shared resource. Include policy is one use of it, not a +separate trust boundary. + +- Managed readable/writable paths are repository-relative and must resolve within + the canonical checkout realpath. Managed path declarations using absolute paths, + `..` escapes, unapproved symlinks, FIFOs, devices, sockets, or other special files + are `INVALID`. +- A protected `RepositoryAlias` may represent an existing Git-tracked base-approved + symlink such as `prompts -> pdd/prompts`. Its logical repository-relative path is + preserved for identity while the manifest records the link blob/hash, canonical + target path/identity, and alias-policy digest. The complete chain must remain + inside the checkout, acyclic, non-dangling, and unchanged from protected base. + Candidate alias additions/retargets cannot govern their own check. +- Transactions never write through or replace a symlink. For an approved alias, + writes target the prevalidated canonical regular file through descriptor-relative + operations while rechecking the protected alias chain. Link retarget, dangling or + cyclic aliases, target replacement, and descriptor-time swaps fail closed. +- Protected external roots are read-only inputs only. PDD never finalizes managed + artifacts, metadata, evidence, or journals outside the checkout. +- An explicit CLI output outside managed roots is allowed only as an unowned export: + it is marked `SKIPPED_NO_FINALIZATION`, cannot update the unit fingerprint, and + cannot contribute trusted semantic evidence. +- Resolution uses `lstat`/realpath checks for every existing component. Transaction + commit revalidates with descriptor-relative operations (`openat`/`renameat` style), + no-follow flags, inode/type preconditions, and parent directory descriptors so a + symlink swap after planning becomes `CONFLICT`/failure rather than an overwrite. +- Capability-registry read/write scopes are constrained by `PathPolicy`; declaring a + broad scope cannot widen protected roots. +- The policy digest is part of the manifest and trusted evidence. Candidate changes + are proposals only and cannot govern the check evaluating that branch. + +#### Language and format identity + +Replace spelling/extension inference with a protected `LanguageSpec` registry. Each +row has a stable unique `language_id`, canonical display name, explicit aliases, +role-specific extensions/formats, text/binary hashing mode, and generator/runner +capabilities. Each language/role also references an `ArtifactModePolicy` defining +allowed artifact type, normalized Git mode (`100644`, `100755`, or protected alias +`120000`), and policy-relevant permission bits. `UnitId` and fingerprints persist +`language_id`; the registry/mode-policy digest is part of the manifest and evidence +closure. + +- Every alias resolves to exactly one language ID in a declared context. Duplicate + normalized aliases, conflicting role extensions, and ambiguous extension-only + inference are `INVALID` until the registry is repaired. +- Extension collisions require explicit language/config identity; the resolver does + not select the first row. +- Existing duplicate/ambiguous rows receive an explicit migration mapping before + v2 identity becomes authoritative. +- Tests are generated from every protected registry row and every applicable output + role in source and installed-wheel environments. Registry additions cannot merge + without their generated conformance cases. + +`UnitSnapshot` records content digest, artifact type, normalized Git mode, and +policy-relevant permission bits for every artifact. Mode-only changes are drift. +Platform-specific permission noise outside the protected policy is normalized, but +the executable bit and every declared security-relevant bit are authoritative. + +### 5.3 Canonical dependency graph + +The include parser used by preprocessing becomes the only parser. It must return +structured include records for: + +- Bare `path`. +- Attributed body forms including `path`, `select`, and `query`. +- Self-closing ``. +- `` with comma and newline separation. +- Backtick include syntax where still supported. +- Transitive includes with cycle detection and a deterministic traversal order. + +Path trust is defined before parsing content: + +- The default readable boundary is the canonical realpath of the repository + checkout, not the nearest nested config root. Nested `.pddrc` files may reference + parent paths that remain inside that repository boundary. +- Resolve and normalize every path and symlink before the boundary check. Relative, + `..`, absolute, and symlinked paths that escape the checkout are rejected. +- A protected-base external-root policy may allow a named external root. It is + mounted read-only in the CI sandbox, assigned a stable external-root ID, and its + exact content snapshot contributes to the baseline/evidence closure. +- Candidate branches cannot add or widen external roots for the check evaluating + that branch. Local commands use the same protected policy and fail closed when + it is unavailable. + +The result is a typed directed multigraph. Every edge records its type, direction, +source authority, propagation policy, ownership effect, and hash contribution: + +- `PROMPT_INCLUDE`: document/input -> prompt; reverse changes trigger regeneration. +- `UNIT_CONTRACT`: provider unit -> consumer unit; public-contract changes propagate. +- `TEST_VALIDATES`: test -> unit; it contributes evidence but not regeneration order. +- `RUNTIME_DEPENDENCY`: provider -> consumer; it informs affected validation and may + have strongly connected components. +- `ARCHITECTURE_ASSERTION`: a declaration that must agree with authoritative prompt + and config inputs; disagreement is `INVALID`, not an extra inferred edge. + +Traversal is edge-policy-specific. Include cycles are errors, while legitimate +runtime dependency cycles are condensed into strongly connected components and +validated as a group. Graph declarations and resolved dependency snapshots are +hashed into the baseline and semantic evidence closure. Missing, ambiguous, +out-of-root, or policy-invalid edges are reportable failures rather than warnings. + +All prompt expansion inputs are represented, including deterministic cached +results from supported web/shell/auto-deps inputs. An uncached or nondeterministic +external input makes semantic status `UNKNOWN` until a reviewed snapshot is bound +to the transaction. + +### 5.4 Snapshot and classifier + +The classifier is a pure function: + +```python +def classify( + unit: ResolvedUnit, + baseline: FingerprintRecord | None, + current: UnitSnapshot, + evidence: ValidationEvidence | None, + git_changes: GitChangeSet | None = None, +) -> SyncVerdict: + ... +``` + +It performs no writes, LLM calls, subprocess calls, prompts, or logging side +effects. `git_changes` refines provenance and rename detection but never overrides +hash truth. + +The minimum classification matrix is: + +| Prompt/docs | Derived artifacts | Baseline result | Semantic result | Default action | +| --- | --- | --- | --- | --- | +| unchanged | unchanged | `CURRENT` | evidence-dependent | none or validate | +| changed | unchanged | `DRIFTED` | `UNKNOWN` | prompt-side repair (`sync`) | +| unchanged | changed | `DRIFTED` | `UNKNOWN` | derived-side repair (`update`/validate) | +| changed | changed | `DRIFTED` | `CONFLICT` | explicit resolution | +| any required file missing | any | `DRIFTED`/`CORRUPT` | `FAILED` | restore or regenerate, never stamp | +| no valid baseline | any | `UNBASELINED` | `UNKNOWN` | audited baseline or verified rebuild | + +Test-only and example-only edits are derived-side changes, but ownership controls +whether PDD may rewrite them. Human-owned tests are validation inputs, never +generated outputs. + +#### Verification profiles + +Trusted execution proves that checks ran; it does not prove those checks cover the +prompt's complete intent. Every managed unit therefore has a typed +`VerificationProfile` with individually addressable obligations: + +```python +@dataclass(frozen=True) +class VerificationObligation: + obligation_id: str + kind: str # interface, requirement, story, example, test, policy, dependency + validator_id: str + validator_config_digest: str + required: bool = True + +@dataclass(frozen=True) +class VerificationProfile: + unit_id: UnitId + obligations: tuple[VerificationObligation, ...] + profile_digest: str +``` + +The profile accounts for every structured prompt requirement/contract identifier, +declared interface, required story/example, PDD-owned and human-owned validation +test, policy/security check, and dependency-closure obligation. A completeness +validator reports unassigned requirements. Non-machine-verifiable requirements +need a protected human-review attestation bound to the same snapshot; they cannot +be silently omitted. + +Profile and validator authority comes from protected-base policy and the trusted +checker release, not the candidate tree alone: + +- Candidate additions form a conservative union with base obligations. Deleting + an obligation, making it optional, remapping its validator, changing validator + config, or changing an obligation-bearing test requires the designated protected + profile/test owner approval. +- A candidate-modified validator implementation cannot validate its own change. + The trusted released implementation remains authoritative until the two-release + checker/validator promotion protocol completes. +- A changed validation test is provisional evidence. Trusted CI runs applicable + unchanged/base obligations as well as candidate tests, and the changed test + cannot be the sole proof of the behavior it was changed to accept. Protected + owner review or another independent obligation is required. +- Profile completeness is evaluated against both base and candidate prompt/ + contract inputs, so deleting a requirement and its mapping together is detected. + +Effective semantic `VERIFIED` requires trusted current evidence for every required +obligation in the protected profile. Interface-only checks, generated-old versus +generated-new comparison, partial profiles, missing validators, unstructured legacy +requirements, and locally authored evidence are safety signals only and leave the +overall unit `UNKNOWN`. A unit without a complete profile cannot enter the managed +sync guarantee: a historically human-owned candidate may remain human-owned, but an +expected-managed unit must be migrated or remain blocked. + +Runner adapters normalize every expected and collected test to one terminal +outcome: `PASS`, `FAIL`, `ERROR`, `COLLECTION_ERROR`, `SKIP`, `XFAIL`, `XPASS`, +`DESELECTED`, `TIMEOUT`, `NOT_COLLECTED`, `QUARANTINED`, or `FLAKY`. Only `PASS` +may satisfy a required test obligation. Zero collected tests, collection errors, +and every other outcome leave the obligation unverified or failed. Diagnostic +retries preserve the first outcome and cannot turn a required failure into trusted +pass; an approved flaky/quarantine exception is a waiver, not `VERIFIED`. +`--skip-tests`, `--skip-verify`, filters, and deselection therefore cannot produce +overall `VERIFIED`. + +### 5.5 Fingerprint schema + +Introduce a versioned schema while reading the legacy format: + +```json +{ + "schema_version": 2, + "hash_algorithm_version": 2, + "unit_id": { + "repository_id": "018f3f5e-7b3c-7f12-a6d4-4b7fd7d8a921", + "prompt_relpath": "pdd/prompts/widget_python.prompt", + "language_id": "python" + }, + "artifacts": { + "prompt": {"path": "pdd/prompts/widget_python.prompt", "hash": "...", "git_mode": "100644"}, + "code": {"path": "pdd/widget.py", "hash": "...", "git_mode": "100644"}, + "examples": {"context/widget_example.py": {"hash": "...", "git_mode": "100755"}}, + "tests": {"tests/test_widget.py": {"hash": "...", "git_mode": "100644"}} + }, + "include_deps": {"docs/widget.md": "..."}, + "manifest_digest": "...", + "config_digest": "...", + "ownership_digest": "...", + "dependency_snapshot_digest": "...", + "verification_profile_digest": "...", + "provenance": { + "kind": "generated|updated|resolved|baseline-reset", + "command": "pdd sync widget", + "transaction_id": "...", + "git_commit": "..." + }, + "claimed_semantic_status": "VERIFIED|UNKNOWN", + "attestation_ref": "...", + "timestamp": "...", + "pdd_version": "..." +} +``` + +The tracked semantic field is an informational projection. A trusted checker +derives the effective semantic status by recomputing validation or verifying a +trusted DSSE/in-toto-style attestation. The attestation binds the complete input +closure, role-to-path mappings, manifest/config/ownership/graph digests, exact +runner commands and config files, validator and PDD versions, model/generator +identity where relevant, result summary, base SHA, and checked head or merge-group +SHA. Candidate-authored evidence without a trusted signature is `UNKNOWN`. + +The fingerprint must not store null hashes for required existing artifacts. +Multi-file tests and examples are maps, not a lossy single representative hash. +Ancestor recovery stores a verified Git-object lookup strategy or durable +content-addressed snapshot sufficient for a three-way merge. PR 4 records the +choice in an ADR, including shallow-clone fetch behavior, retention, and secret +handling, and enables no conflict resolver until recovery works in CI. + +### 5.6 Transaction protocol + +`FingerprintTransaction` owns one resolved unit and participates in a durable +operation journal. Atomic rename of one fingerprint is not described as a +multi-file transaction. + +1. Acquire all unit and shared-resource locks in deterministic global order. + Shared files such as `architecture.json`, ownership policy, and graph snapshots + have their own resource locks; nested calls inherit the owning transaction. +2. Capture compare-and-swap precondition hashes for every input and writable + resource, including artifact type, normalized Git mode, and policy-relevant + permission bits. Resolve paths once through protected `PathPolicy`; callers may + not substitute another root later. +3. Persist and `fsync` a write-ahead journal entry containing participants, + preconditions, durable rollback blobs, planned paths, and prepared output hashes. +4. Execute generators/tests in isolated staging locations. No owned destination is + overwritten during preparation. +5. Validate outputs and create provisional run/evidence records tied to prepared + hashes. +6. Recheck all preconditions. An external edit since planning changes the verdict + to `CONFLICT`; it is never overwritten. Reopen destination parents through + descriptor-relative no-follow operations and reject inode/type/mode/permission/ + symlink changes using `fstat`. +7. Mark the journal `COMMITTING`, then install artifacts and shared-file patches in + deterministic order using same-directory temp files, flush/`fsync`, `os.replace`, + and directory `fsync`. Set and verify the protected destination mode on staged + files before install. Shared-file patches use compare-and-swap merge semantics. +8. Persist run reports and the fingerprint, then mark the journal `COMMITTED` and + release resources. +9. At mutator preflight, or through explicit `pdd recover --transaction ID`, + recovery scans incomplete journals. Based on the durable commit marker it + deterministically completes the prepared commit or restores all participants + from durable rollback blobs. Recovery itself is idempotent. Read-only commands + never recover implicitly; they return `RECOVERY_REQUIRED` with the exact + recovery command and leave all bytes unchanged. +10. Any failure raises a typed non-zero command error. Finalization failure is never + downgraded to a warning. + +Multi-unit commands maintain an aggregate journal with one state per participant. +Policy explicitly chooses all-or-nothing or independently committable units. In +the latter mode, a later failure reports `PARTIAL` with committed and pending unit +IDs; it never reports global success or loses the ability to resume. Resource-lock +ordering and aggregate recovery prevent deadlocks and shared-file lost updates. + +Journal/rollback storage is operational state, never repository content: + +- Store it under a Git-excluded per-repository state directory with directory mode + `0700` and files `0600`; CI uses an ephemeral encrypted workspace. +- Apply bounded byte/count/age retention and clean committed/recovered journals. + Retain unresolved journals until explicit recovery or protected quarantine. +- Secret-labeled artifacts require platform-backed encryption at rest or the + transaction refuses to persist rollback content. +- Rollback records preserve artifact type and protected mode/permission metadata; + recovery restores and verifies them, not only content bytes. +- Preflight available space before preparation. Quota/disk failures abort before + destination writes and preserve the prior state. +- Artifact upload and debug-log collectors explicitly exclude journals, rollback + blobs, and staged prompt/code content. +- Tests prove journals cannot be staged by normal Git commands, included in built + packages, or uploaded by CI artifact patterns. + +Dry-run and redirected-output modes explicitly record `SKIPPED_NO_FINALIZATION` +and cannot be mistaken for successful synchronized transitions. + +### 5.7 Repair planner + +The planner accepts a verdict and policy, then returns allowed actions. It never +changes classification to make a preferred operation possible. + +| Verdict | Allowed unattended action | Interactive/reviewed action | +| --- | --- | --- | +| Prompt/doc-only drift | targeted `sync` if policy permits generation | review generated patch | +| Code-only drift | none by default; only a proven requirement-preserving three-way update under a complete profile | review and approve minimal prompt patch | +| Test/example-only drift | validate; update prompt only when contract changed | adopt or keep human-owned artifact | +| Conflict | none | prompt wins, code wins, or three-way merge | +| Unbaselined | none | verified rebuild or explicit baseline reset | +| Missing/corrupt artifact | restore/regenerate only when source is unambiguous | manual recovery | +| Validation failure | bounded repair with circuit breaker | manual fix | + +`reconcile --heal` is deprecated because it currently means "accept current +hashes." Replacement commands are explicit: + +- `pdd reconcile`: read-only status/report. +- `pdd reconcile --check`: read-only non-zero gate. +- `pdd baseline --accept-current UNIT --reason TEXT`: reviewed baseline reset, + resulting in semantic `UNKNOWN`. +- `pdd resolve UNIT`: explicit conflict workflow. +- `pdd sync` / `pdd update`: semantic repair workflows. + +Code-to-prompt repair is a three-way minimal patch using the last verified prompt +as ancestor. It must preserve every existing requirement/contract identifier and +all unrelated text, then satisfy every obligation in the complete verification +profile. If the prompt is unstructured, preservation cannot be proved, or any +obligation lacks trusted evidence, automation emits a review patch and leaves the +unit `UNKNOWN`; it does not finalize unattended. + +## 6. Delivery workstreams and PR sequence + +Each PR must be independently reviewable. Do not merge a 200-file epoch stamp in +the same PR as resolver, classifier, command, or policy changes. + +### PR 0: Restore governance and pin strict expected failures + +Issues: #1932, #1836, #1927-#1931 + +Tasks: + +- Reopen #1932 or create a replacement tracking epic that links every workstream. +- Record PR #1954 as a partial milestone. +- Add this plan to the epic and make its exit gates the Definition of Done. +- Convert the existing issue #1932 tests into a durable acceptance lane. +- Add reproductions for the audit findings: incomplete inventory, semantic status + after baseline reset, current/live classifier divergence, and missing artifact + behavior. +- Put unresolved acceptance tests in a dedicated nonblocking lane with + issue-linked `pytest.mark.xfail(strict=True)`. A fix that unexpectedly passes + fails as XPASS until the marker is removed and the test moves to the required + lane; ordinary required CI stays green while PR 0 lands. +- Keep PR #1969 blocked and split it along the PR boundaries below. +- Close or supersede PR #1937 unless it is completed as PR 4/5 below. + +Exit gate: + +- The epic is open, child dependencies are visible, and strict expected-failure + tests reproduce each unresolved guarantee without changing production behavior. + +### PR 1: Canonical unit manifest and ownership + +Issues: #641, #884, #1931, #1903 + +Tasks: + +- Add `CandidateId`, `UnitId`, `SubjectId`, `ResolvedUnit`, ownership types, and + manifest diagnostics. +- Implement the resolution precedence in section 5.2. +- Build the candidate union from base/head manifests, configured source and test + roots, prompts, architecture entries, fingerprints, ownership records, and + tracked artifacts. Enumerate unmatched code/tests and whole-unit deletions even + when no prompt or metadata remains at head. +- Represent hand-maintained code/tests and intentional orphans explicitly. +- Add explicit rename migrations and reviewed removal tombstones. +- Keep the inventory lock generated and non-authoritative; changing source inputs + without regenerating it fails the assertion. +- Generate the protected expected-managed assertion from base/history provenance; + prohibit debt-clearing ownership demotions. +- Port the useful architecture completeness work from PR #1969 without the epoch + stamp. +- Exclude build-generated files such as `_version.py` by ownership/source rules, + not environment-specific waivers. +- Add collision detection for flattened metadata names and duplicate leaves. +- Apply one protected `PathPolicy` to every managed read/write role and emit typed + diagnostics for absolute paths, escapes, symlinks, and special files. +- Inventory current tracked in-repository symlink aliases in PDD and pdd_cloud and + add protected `RepositoryAlias` records without changing logical unit identity. +- Replace ambiguous language spelling/extension inference with stable + `LanguageSpec.language_id`, repair duplicate aliases, and bind the registry + digest to the manifest. + +Tests: + +- Flat, path-qualified, nested `.pddrc`, worktree, duplicate leaf, architecture + override, missing prompt, missing code, rename, and deletion. +- Every protected language registry row across every applicable output role, + including extension collisions, duplicate aliases, source/wheel parity, and + explicit disambiguation. +- Approved tracked aliases, candidate retarget, dangling/cycle, canonical target + replacement, descriptor-time swap, and source/wheel identity parity. +- Source checkout and built/installed wheel produce identical manifests. +- PDD repository inventory count is deterministic and every exclusion has a + structured reason. +- Base-to-head whole-unit deletion and config retarget to same-content paths are + detected. +- An independent `git ls-files` plus base-tree partition accounts for every tracked + path as managed artifact, validated artifact, configuration/policy, explicit + exclusion, or unrelated human-owned path. Compare this partition to resolver + output so resolver and generated lock cannot share an omission. + +Exit gate: + +- `managed + human_owned + removed + invalid == all discovered candidates` + in all supported invocation environments, and the independent Git-tree partition + has no unaccounted path; no silent omissions or debt-clearing ownership demotions. + +### PR 2: Canonical includes, graph, and hashing + +Issues: #881, #1807, #1930 + +Tasks: + +- Extract/reuse the preprocessing include parser as a structured API. +- Replace include regexes in sync ordering, fingerprints, update scope guards, + architecture validation, and CI detection. +- Build the typed deterministic multigraph with source attribution, edge-specific + reverse propagation, and SCC policy. +- Make prompt composite hashing consume the graph result. +- Preserve stored include identities when preprocessing removes source tags. +- Add hash schema support for multiple tests/examples. +- Include manifest, config, ownership, graph declarations, dependency snapshots, + and cached external expansion inputs in baseline/evidence closure. +- Include artifact type, normalized Git mode, and protected permission policy in + snapshots, hashes/comparisons, baseline, and evidence closure. + +Tests: + +- Every supported include syntax, transitive closure, cycle, missing include, + outside-root include, changed dependency, stripped tag, and deterministic order. +- Golden parity against current valid fingerprints and pdd_cloud include-bearing + units. + +Exit gate: + +- Exactly one include parser and one composite prompt hash builder remain in + production code. Edge types cannot be traversed without an explicit propagation + policy. + +### PR 3: Pure shared classifier and conformance matrix + +Issues: #884, #1931, #1838 + +Tasks: + +- Add snapshot, evidence, verification-profile, obligation, verdict, and stable + report-schema types. +- Implement the pure classifier and full state matrix. +- Treat candidate-authored evidence as untrusted. Add an attestation-verifier + boundary; only a trusted validator result can derive effective `VERIFIED`. +- Add profile completeness and effective-semantic-status evaluation. Partial or + legacy-only safety checks cannot yield overall `VERIFIED`. +- Model missing/corrupt fingerprints and artifacts without fallback success. +- Make Git diff/rename data an explicit optional classifier input. +- Add a compatibility adapter for `sync_determine_operation`. +- Route read-only modes of sync, update, change, CI heal, agentic sync, and global + scan to the classifier. +- Remove CI-only reclassification branches once parity tests pass. +- Inventory every inline bug-reference comment in `sync_determine_operation.py`; + link a regression test or file a live issue for each. + +Tests: + +- Cross-product/property tests over artifact existence, hash changes, baseline + validity, evidence freshness, ownership, and Git change provenance. +- Conformance test runs every consumer against the same fixture and compares the + complete verdict, not only a classification string. +- Failure injection proves exceptions become typed `FAILED`, never skips. +- Forged evidence and manifest/config retargets cannot produce `VERIFIED` or + `CURRENT`. + +Exit gate: + +- All read-only consumers return byte-identical normalized verdicts for the same + input. `sync_determine_operation.py` contains no independent hash or drift + classification logic. + +### PR 4: Atomic fingerprint store and schema migration + +Issues: #1926 + +Tasks: + +- Add schema and hash-algorithm versions plus the v2 reader/writer. +- Persist artifact type/Git mode and mode-policy version for every role/path. +- Record an ADR for ancestor recovery and metadata-path migration before enabling + v2-dependent conflict behavior. +- Add per-unit file locks and atomic replace with flush/fsync semantics. +- Reject null required hashes and identity/path mismatch. +- Preserve old fingerprints until a successful atomic replacement. +- Add ancestor reference/snapshot support required by conflict resolution. +- Use a collision-safe v2 metadata path. During migration, preserve legacy + top-level fields or a generated v1 projection for old readers, but keep canonical + v2 state in a path old writers cannot overwrite. +- Add a protected repository minimum-writer version policy. Trusted automation + fails closed below it; new checkers detect and block legacy projection writes + from direct old local binaries. +- Define forward migration and rollback. Do not make v2 authoritative until all + trusted writers satisfy the minimum version. +- Reject mutable owner/repository slugs and absolute paths in v2 identity schema; + require the protected repository UUID, stable language ID, and + repository-relative POSIX paths. +- Make direct writes outside `fingerprint_store.py` fail a static architecture + test. + +Tests: + +- Crash before write, partial temp write, replace failure, permission failure, + concurrent writers, stale lock, nested call, wrong root, malformed JSON, legacy + migration, old/new writer races, collision migration, rollback, disk exhaustion, + and v2 UUID/path schema validation. + +Exit gate: + +- One internal fingerprint writer exists; failed persistence cannot corrupt or + replace the prior valid fingerprint. + +### PR 5: Transactional finalization for every mutator + +Issues: #1926, #356, #1836 + +Tasks: + +- Add `FingerprintTransaction`, durable per-operation write-ahead journals, + aggregate multi-unit state, startup recovery, ordered shared-resource locking, + compare-and-swap preconditions, and typed finalization failures. +- Route generate, example, auto-deps, update single/all, fix, sync, change, + metadata sync, CI heal, test generation, architecture sync, server execution, + agentic entry points, and every other registered mutator through it. +- Add a command capability registry covering every CLI, server, CI, and agentic + entry point. Each record declares read paths, managed/shared write scopes, + ownership effects, transaction mode, validation obligations, sandbox policy, + and whether partial multi-unit commit is allowed. +- Derive the mutator invariant suite from the registry. Add a static/runtime write + guard in tests so an unregistered entry point or an undeclared managed/shared + write fails CI. Future commands cannot merge without a capability record. +- Remove or delegate every direct `save_fingerprint` call. +- Tie run reports and evidence to current input/output hashes. +- Prevent success summaries and zero exits when finalization fails. +- Ensure `pdd fix` regenerates/finalizes code when it changes a prompt. + +Tests: + +- Parametrized post-command invariant for every mutator. +- Registry completeness includes `pdd test`, `sync-architecture`, all command + aliases, server routes/executors, CI scripts, and agentic subprocess launchers. +- Failure and process death at each transaction stage recover according to the + durable commit marker without overwriting external edits. +- Mode-only precondition changes conflict; staged install and rollback preserve and + verify protected executable/permission bits with descriptor `fstat`. +- Multi-unit runs expose committed/pending participants and resume or roll back + according to their declared aggregate policy. +- Concurrent shared `architecture.json` updates cannot lose either valid change. +- Output-path escapes and symlink-swap races fail without modifying in-repository + or external bytes; explicit out-of-root exports remain unfinalized. +- Journal tests cover restrictive permissions, disk/quota preflight, bounded + retention, committed/recovered cleanup, encryption-required refusal, Git/package + exclusion, and CI artifact/debug-log exclusion. +- Every successful command is a no-op on immediate second execution. + +Exit gate: + +- No mutating path can bypass transactional finalization, and the property test is + required CI. + +### PR 6: Honest reconcile, baseline, and repository inventory gate + +Issues: #1927, #1836 + +Tasks: + +- Make `pdd reconcile` read-only by default with stable human/JSON output. +- Add `--check`, module scoping, affected-only scoping, and strict exit semantics. +- Deprecate `--heal`; do not silently map it to a baseline reset. +- Add explicit `pdd baseline --accept-current --reason` and `--backfill` flows. +- Mark accepted/backfilled units semantic `UNKNOWN` until validation succeeds. +- Run the repository inventory/fingerprint check in nonblocking shadow mode. Do + not port the 223-unit data baseline or activate a required gate in this PR. + +Exit gate: + +- A clean source checkout and installed wheel enumerate the same units and produce + the same shadow report. Baseline reset cannot produce `IN_SYNC` without trusted + evidence, and no check blocks before the audited baseline exists. + +### PR 7: Explicit conflict resolution + +Issues: #1929, #883 + +Tasks: + +- Keep the merged protection that preserves fingerprints/run reports on co-edit. +- Persist a structured resolution plan and ancestor provenance. +- Implement `pdd resolve UNIT --prompt-wins`, `--code-wins`, and `--merge`. +- Show prompt-side and derived-side diffs from the last verified ancestor. +- Require explicit confirmation for destructive choices in interactive mode. +- Require a reason flag in non-interactive reviewed automation. +- Run normal validation and transactional finalization after resolution. +- Remove placeholder/stub strategies before command exposure. + +Exit gate: + +- Four-quadrant tests pass in every consumer; conflict E2E proves both edits and + ancestor metadata survive unattended automation. + +### PR 8: Semantic contract and self-repair + +Issues: #1900, #1837, #1839, #928-class regressions + +Tasks: + +- Parse prompt-declared interfaces into a typed contract. +- Parse or assign stable identifiers to structured behavioral, security, + operational, story, and policy requirements; build a complete + `VerificationProfile` and report every unassigned requirement. +- Validate generated code against the prompt contract, not generated-old versus + generated-new alone. +- Keep legacy surface comparison only when no prompt contract exists. +- Treat that legacy comparison as a safety guard only; it cannot produce overall + semantic `VERIFIED`. +- Add bounded repair attempts with stable signatures and a real circuit breaker. +- Emit complete expected/actual diagnostics on final failure. +- Preserve unrelated declared symbols during incremental generation. +- Make code-to-prompt update a three-way minimal patch against the last verified + prompt. Preserve all existing requirement/contract IDs and unrelated text. + Unstructured or unprovable changes produce a review patch and remain `UNKNOWN`. +- Treat contract validation evidence as part of semantic `VERIFIED`. +- Migrate managed units to complete profiles. Units without complete profiles stay + `UNKNOWN` and cannot enter enforcement as managed. + +Exit gate: + +- Additive declared changes pass; undeclared removals/signature changes repair or + fail honestly; historical false-positive and dropped-feature fixtures pass. No + incomplete profile or legacy-only comparison yields overall `VERIFIED`, and + code-to-prompt updates preserve all unrelated requirements byte-for-byte. + +### PR 9: Included-doc propagation and affected-unit repair + +Issues: #890, #1930, #881 + +Tasks: + +- Compute affected closures using edge-specific propagation rules from the typed + graph. Do not topologically order test-validation edges as generation edges. +- Permit included-doc edits through update/change scope guards. +- Reject unrelated edits and cross-unit writes outside the plan. +- Re-sync every affected managed unit in topological order. +- Reject include cycles before mutation; condense allowed runtime dependency SCCs + and validate them as groups. +- Finalize each unit transactionally and retain a durable multi-unit plan. + +Exit gate: + +- Historical scope-guard regressions #880/#987/#1025/#1054 are permanent E2E + fixtures; second propagation run is a no-op. + +### PR 10: Test ownership and runner alignment + +Issues: #890, #1903, #1917 + +Tasks: + +- Add explicit runner adapters (`pytest`, Jest, Vitest, and repository-declared + adapters) with runner ID, working directory, collection command, execution + command, config paths, and config digest. Support multiple disjoint runners. +- Prove collection through each runner's own collection mechanism; path inference + alone is insufficient. +- Distinguish generated/PDD-owned tests from human-owned validation tests. +- Record ownership per artifact with provenance. Legacy tests default to + human-owned unless an explicit reviewed adoption changes ownership. +- Adopt existing tests only through an explicit reviewed ownership transition. +- Never fork a shadow test outside the runner's collection path. +- Never overwrite human-owned tests. +- Let test churn open a review item/PR rather than corrupting or silently skipping + validation. Bind evidence to the exact runner command, working directory, + configuration digest, and collected test IDs. +- Require every test in `validated_artifacts`, regardless of ownership, to be + collected and executed by a declared runner adapter or carry an exact temporary + waiver. A runner-config change that drops or reroutes a validation test changes + the manifest/profile digest and invalidates prior evidence. +- Emit normalized terminal outcomes for expected and collected IDs. Enforce + pass-only obligation semantics, zero-test failure, fail-closed collection, + skip/xfail/xpass/deselection/timeout handling, and non-overriding diagnostic + retries. + +Exit gate: + +- Every PDD-owned and human-owned validation test is collected and executed by its + declared real project runner. Human-owned tests remain byte-identical unless + explicitly adopted, and no runner/config change can silently remove them from + evidence. Every required test obligation is `PASS`; unresolved xfails, skips, + deselections, quarantine, collection errors, timeouts, and flaky outcomes are 0. + +### PR 11: Ledger and local boundary hooks + +Issues: #1928 + +Tasks: + +- Store a deduplicated durable ledger keyed by `SubjectId`, trigger, and current + artifact hashes. +- Make editor/agent hooks map touched paths to units without LLM/network calls. +- Install, update, and uninstall Git hooks idempotently while preserving existing + user hooks and honoring `core.hooksPath`/worktrees. +- Classify only commit-touched units under the default fast path. +- Default local policy annotates; optional strict policy blocks. +- Add ledger claim/ack semantics so successful CI/repair clears only entries it + actually resolved. +- Treat the ledger as an advisory, reconstructable cache. Corruption, loss, + expiration, rename, or absence can reduce targeting performance but can never + suppress the full PR/merge/nightly scan. Rebuild it from Git diff plus current + snapshots and update shared cursors with compare-and-swap. + +Exit gate: + +- Edit -> hook -> ledger -> targeted repair -> acknowledgement E2E passes in + under the declared performance budget and leaves unrelated entries intact. + +### PR 12: PR, merge, and nightly checks in shadow mode + +Issues: #1928, #1836, #1464 + +Tasks: + +- Add a nonblocking shadow PR check that classifies changed/affected units and + merge-group synthetic commits. Enforcement activates only in PR 14. +- Load enforcement policy and waiver authority from the protected base branch or + an external control plane, never from the candidate tree alone. +- Run the released/base trusted checker alongside candidate code when the PR + modifies the classifier, policy loader, schemas, or validation adapters. +- Define the future blocker set now: baseline `DRIFTED`, `UNBASELINED`, `CORRUPT`; + semantic `UNKNOWN`, `CONFLICT`, `FAILED`; inventory `INVALID`; and any manifest + removal/retarget lacking a valid migration. +- Define structured waivers bound to subject/verdict/snapshot/manifest digests, + issuance provenance, affected dependency-closure digest, owner, protected approver + identity, reason, issue, creation time, and expiry. +- Keep waivers orthogonal to inventory. A managed unit with a waiver remains in + gross managed and non-`IN_SYNC` counts; it is never reclassified as `WAIVED`. +- Treat ownership, adoption, repository-ID, rename, and removal changes as + protected policy transitions loaded from the base and requiring CODEOWNERS + approval. +- Treat verification-profile obligations/requiredness/mappings, validator + implementations/config, and obligation-bearing test changes as protected + transitions. Candidate changes cannot weaken the base obligation union or + certify themselves. +- Re-run the strict check after any bot repair commit. +- Design checks for the final head SHA, current base SHA, and merge-group SHA, not + an earlier successful head. +- Run a full read-only reconcile after merge and nightly. +- Nightly automation opens/updates a rolling repair PR; it never baseline-resets + `main` automatically. +- Persist last-success cursors so failed nights do not create time-window gaps. +- Treat cursors as compare-and-swap advisory caches reconstructable from Git; a + missing/corrupt cursor triggers a safe full scan. +- Define an untrusted-code boundary: classification is read-only; model and test + execution run in an ephemeral sandbox without production secrets or push token, + with CPU/time/output/filesystem/network limits. Repair produces a patch artifact. + A separate narrow trusted applier checks allowed paths and head-SHA CAS before + applying/pushing, then reclassifies the new head. +- Deploy the evidence trust plane before PR 13: + - Protected-base/control-plane `AttestationTrustPolicy` loader and verifier. + - Post-validation signer using dedicated workload identity and no candidate code. + - Authenticated trusted-runner result channel plus a nonce/check-run authority. + - Transparency/audit record store and evidence cache invalidation service. + - Threshold protected human-review attestation workflow for non-machine-verifiable + obligations. + - Rotation, revocation, expiry, replay rejection, outage, and compromise response. +- The signer does not sign sandbox-submitted JSON. It independently authenticates + the trusted runner/workflow, fetches or recomputes observed hashes and profile/ + manifest/config/runner digests, verifies the check-run base/head/merge-group SHA + and nonce, and signs only the resulting canonical statement. +- Run staging issuance and verification E2E for machine and threshold-human + obligations, unknown signer, replay, signer outage, key/root rotation, revocation, + cached-evidence invalidation, and compromised-signer recovery. + +Exit gate: + +- Shadow reports cover manual, agent, PDD, hotfix, base-movement, and merge-group + channels with zero unexplained differences from expected enforcement. Sandbox, + trusted-applier, stale-head, and policy-tamper tests pass; the evidence trust plane + passes staging issuance/replay/outage/rotation/revocation/compromise E2E; merge is + not blocked yet. PR 13 cannot start before this gate. + +### PR 13: PDD dogfood baseline and burn-down + +Tasks: + +- Generate the canonical PDD inventory lock and ownership/waiver assertions. The + lock is derived output, not an authoritative registry. +- Classify all units before creating a new baseline. +- Build complete verification profiles for every managed unit. Migrate structured + requirements for every expected-managed unit. Historically human-owned artifacts + remain separate; do not use ownership demotion or waivers to avoid profile work. +- Divide legacy units into verified, conflict, invalid, and accepted-unknown. +- Repair verified candidates through normal commands. +- Resolve conflicts with review. +- Baseline-reset only units whose current tree is explicitly accepted. Each such + unit remains semantic `UNKNOWN` and must gain trusted validation evidence before + enforcement. Expected-managed debt cannot be reclassified as human-owned. +- Commit the baseline in a data-only PR after all implementation PRs are green. +- Do not enable enforcement in the data PR. + +Exit gate: + +- PDD `main` reports complete inventory and every managed unit is full trusted + `IN_SYNC`. Gross managed count is non-zero and fixed by the candidate union; + managed waivers and semantic `UNKNOWN` units are zero. Human-owned/removed + candidates and managed-to-human transitions are separately enumerated, with zero + debt-clearing transitions. There are no second-run writes in source and wheel + environments. + +### PR 14: Activate protected enforcement + +Tasks: + +- Change the protected-base policy from `report` to `enforce` in a separately + reviewed operational PR after PR 13's baseline and all shadow gates are green. +- Require the PR 0 issue-linked expected-failure count to be 0; all resolved tests + have had strict xfail markers removed and moved to required lanes. +- Require CODEOWNERS/protected approval for checker policy, waiver authority, + minimum-writer version, runner adapters, sandbox policy, repository identity, + expected-managed/ownership/adoption, rename/removal, verification profiles, + validator implementations/config, obligation-bearing validation tests, + repository aliases, language registry, and attestation trust/issuer policy. +- Require both the trusted released/base checker and candidate checker during the + checker transition window. +- Use the two-release checker promotion protocol in section 8.2; combine old/new + verdicts conservatively so neither checker can downgrade the other's blocker. +- Gate the current base/head merge result or merge-group SHA and disable unaudited + admin bypass for the sync check. +- Enable same-repository repair only through sandbox -> patch artifact -> trusted + applier -> head-SHA CAS -> recheck. Fork repair remains non-mutating and blocked. +- Verify post-merge and nightly full scans cannot be narrowed by ledger/cursor loss. + +Exit gate: + +- Manual, agent, PDD, hotfix, base-movement, and merge-queue channels cannot land a + managed non-`IN_SYNC` unit without a valid protected waiver. Policy-tamper and + forged-evidence PRs are blocked by the trusted checker. + +### PR 15: pdd_cloud migration and upstream reconciler retirement + +Tasks: + +- Pin a released PDD version containing PRs 1-14. +- Compare upstream results with pdd_cloud's vendored prompt reconciler in report + mode across candidate IDs, unit identity, dependencies, affected closures, + artifact paths, hashes, and verdicts; differences are failures to investigate, + not values to average. +- Inventory all pdd_cloud units across nested `.pddrc` and architecture roots. +- Reclassify the historical 377/788 stale population into repair, conflict, + invalid, waived, and accepted-unknown buckets. +- Burn down in bounded, reviewable PRs by project/subtree. +- Retire every vendored sync-semantic component only after parity is zero and the + upstream command meets runner performance/reliability needs: repository/unit + identity, candidate discovery, include parsing, dependency graph, reverse + affected closure, path resolution, hashing, and classification. pdd_cloud may + retain orchestration/scheduling only and consumes the upstream manifest, graph, + closure, and verdict JSON contracts. +- Run one staging nightly, one same-repo repair PR, one manual hotfix PR, and one + conflict canary before enabling enforcement broadly. +- Keep all repair/test/model execution inside the untrusted-code sandbox and use a + separate narrow applier credential. +- Mirror the PDD sequence: shadow report, audited data-only baseline, then a + separately protected enforcement activation. Accepted-unknown units must be + trusted-verified before activation; ownership demotion and managed waivers are + not baseline burn-down strategies. + +Exit gate: + +- pdd_cloud has complete candidate-union inventory, every managed unit trusted + `IN_SYNC`, zero managed waivers and non-`IN_SYNC` managed states, no vendored + identity/discovery/include/graph/closure/path/hash/classifier semantics, and + consecutive full nightly no-op reports. + +## 7. End-to-end validation program + +### 7.1 Test levels + +| Level | Purpose | Network/LLM | +| --- | --- | --- | +| Unit | Pure identity, include, hash, classifier, planner, schema | none | +| Component | Transaction failures, command adapters, ledger, hooks | none | +| Lifecycle fixture | Real Git repo and CLI subprocesses across all boundaries | none or fake model | +| Package | Built wheel in clean environment | none | +| Staging | Real CI bot, branch, check, repair PR, and selected LLM path | bounded | +| Consumer canary | Real pdd_cloud subtree and nightly runner | bounded | + +Tests labeled E2E must cross at least one real process/repository boundary and +must exercise the live mutation path. Calling five reporting wrappers that all +invoke one helper is a conformance test, not lifecycle E2E. + +### 7.2 Required lifecycle matrix + +Each row runs through edit, Git commit, PR classification, repair or resolution, +transactional finalization, merge gate, post-merge check, and nightly no-op. + +| Change source | Expected behavior | +| --- | --- | +| Included doc only | all affected units found and repaired in graph order | +| Prompt only | generated artifacts updated, contracts/tests current | +| Code only | prompt updated or PR blocked; no blind stamp | +| Example only | ownership-aware validation/update | +| Test only | real runner collects it; ownership preserved | +| Prompt + code | conflict, no unattended writes | +| Prompt + test | conflict when test is derived; ownership-aware otherwise | +| Code + validating test | changed test cannot solely certify changed code; protected profile/test policy applies | +| Prompt + code + test | conflict/resolution plus complete independent obligation evidence | +| Rename/move | identity preserved or explicit migration required | +| Required artifact deletion | failure/repair, never current | +| Chmod-only/executable-bit drift | baseline drift detected even when bytes match | +| Concurrent mode change during transaction | CAS/fstat blocks overwrite and preserves external mode | +| Crash recovery with executable artifact | content and protected mode restored/committed together | +| Missing fingerprint | unbaselined, explicit verified rebuild/baseline required | +| Corrupt fingerprint | corrupt failure, old bytes preserved | +| Interrupted mutation | rollback and non-zero exit | +| Process death after each journal phase | startup recovery completes or restores according to durable marker | +| Read-only check sees incomplete journal | reports `RECOVERY_REQUIRED`, exit 1, and performs no writes | +| Two concurrent syncs | one serialized result, no lost update | +| Concurrent shared architecture writes | both valid patches survive or one becomes a conflict | +| Nested `.pddrc` | correct project root and metadata location | +| Duplicate basename | path-qualified identity, no guessed target | +| Approved tracked prompt alias | logical identity stable; canonical target hashed and safely used | +| Candidate alias retarget/dangling/cycle | protected alias policy blocks without writes | +| Alias descriptor-time swap | no-follow canonical-target revalidation blocks commit | +| Every language/role registry case | source and wheel resolve the same stable language ID and paths | +| Ambiguous language alias/extension | inventory `INVALID`; no first-row inference | +| Whole-unit deletion | base manifest/tombstone check blocks silent disappearance | +| Config retarget to identical bytes | manifest drift detected despite equal content hashes | +| Managed-to-human demotion during debt | protected expected-managed policy blocks it | +| Required obligation deleted/remapped | base/candidate profile union remains blocking | +| Validator weakened with its tests | released validator cannot self-certify replacement | +| Human-owned test | validated but never overwritten | +| Multiple test runners | each owned test is collected by its declared adapter; no shadow test | +| Bot repair commit | final head SHA rechecked before merge | +| Stale bot head/rebase | compare-and-swap rejects push and recomputes verdict | +| Merge-group synthetic commit | trusted check binds result to merge-group and base SHAs | +| Forged fingerprint/evidence | trusted validator ignores it and reports `UNKNOWN`/failure | +| Unknown/replayed/expired attestation | trust policy rejects it; obligation remains unverified | +| Attestation key/root rotation | overlap accepts intended issuers only; retired key is rejected | +| Compromised signer revocation | cached evidence invalidated and obligations rerun before `VERIFIED` | +| PR weakens checker policy | protected-base checker and policy still enforce | +| Old/new writer race | canonical v2 state survives; mixed-version check fails closed | +| Shallow history ancestor lookup | bounded fetch recovers ancestor or resolution stops safely | +| Sandbox escape/resource abuse | no secrets/push token exposed; limits terminate the job | +| Nested config parent include within repo | allowed and hashed under repository boundary | +| Absolute/symlink include escape | blocked unless protected external-root policy applies | +| Protected external include | read-only mounted snapshot is bound to evidence | +| Absolute artifact output in config | managed resolution is `INVALID`; no external write | +| Artifact symlink-swap race | descriptor-relative revalidation blocks commit and preserves bytes | +| Artifact FIFO/device/socket path | path policy rejects before execution | +| WAL disk/quota/retention failure | no destination write; secure state retained or cleaned per policy | +| Required test skipped/xfail/deselected | obligation is not verified; gate fails after activation | +| Zero tests/collection error/timeout | normalized failure; retry cannot convert it to trusted pass | +| Failed nightly | next run resumes from last success without a window gap | + +### 7.3 Historical regression fixtures + +Keep minimal, checked-in reproductions for: + +- Null/partial fingerprints and repeated heal loops (#437/#1305 class). +- Wrong subproject root (#1211/#1290 class). +- Rich include syntax missed by fingerprints (#881). +- Included-doc scope guard reverting legitimate edits + (#880/#987/#1025/#1054). +- Public-surface false positive and silent feature loss (#1900/#928 class). +- Input-too-large or detector failure reported as no work (#1838). +- Test written outside the real runner path (#1903/pdd_cloud#3024). +- Prompt and code co-edit with preserved ancestor (#1929). +- Manual code hotfix followed by stale-prompt regeneration risk + (pdd_cloud#2252/#2834 class). + +### 7.4 Required CI lanes + +1. Fast pure-core tests on every PR. +2. Classifier conformance matrix on every sync-related PR. +3. Mutator finalization property suite on every mutator-related PR. +4. Lifecycle fixture suite on every sync/CI/hook/identity change. +5. Built-wheel manifest and reconcile parity on every release candidate. +6. Full repository strict reconcile after unit tests. It is shadow-only before PR + 14 and required before a PR is mergeable after enforcement activation. +7. Staging canary before enabling a new repair policy in production. +8. At least one journal-recovery, forged-evidence, stale-head, and sandbox failure + injection canary per release candidate. + +Path filters may add fast-fail lanes, but the full suite remains part of normal CI; +path filters must never be the only protection against shared-core regressions. + +Lifecycle tests use stable command/check semantics: exit `0` means the trusted +checked snapshot is clean or a requested repair committed successfully; exit `1` +means an unresolved/blocked/validation/transaction condition; exit `2` is reserved +for CLI usage/configuration errors. Every failure-injection row asserts the exact +exit code, pre-existing destination bytes, durable journal recovery state, complete +verdict, and the base/head/merge-group SHA recorded on the check run. A test cannot +pass by asserting only log text. + +## 8. Rollout and migration controls + +### 8.1 Feature flags + +Use repository configuration with explicit defaults: + +```yaml +sync: + classifier_v2: report + transaction_v2: off + conflict_policy: block + boundary_mode: report + nightly_mode: report + baseline_accept_requires_reason: true +``` + +The boundary state machine is `off -> report -> repair -> enforce`. `repair` means +the sandbox may propose a patch but the check remains nonblocking; `enforce` means +the final trusted verdict blocks and may also use the approved repair pipeline. +Rollback moves one state backward and never rewrites baselines. Classifier and +transaction flags have their own `off -> report -> enforce` transitions. Move one +state at a time after its exit gate; do not combine classifier, transaction, +baseline, repair, and enforcement activation. + +### 8.2 Compatibility + +- Read legacy fingerprints throughout one deprecation window. +- Introduce `schema_version` and `hash_algorithm_version` before v2-dependent + behavior. Use a collision-safe canonical v2 path plus a generated legacy + projection during the mixed-version window. +- Enforce a protected minimum writer version for trusted automation. Old direct + local writers cannot overwrite canonical v2 state; any legacy projection change + is detected and blocked by the new checker. +- Make v2 authoritative only after mixed-version source/wheel/rollback tests pass + and all trusted writers meet the minimum. +- Keep existing CLI options with deprecation warnings where behavior is safe. +- Make unsafe `--heal` fail with actionable migration text before removal. +- Version the JSON report schema and provide a compatibility projection for CI + consumers during migration. +- Package all runtime modules and schemas in the wheel; package tests verify it. +- Document forward migration, emergency rollback, and how canonical v2 state + regenerates the legacy projection after rollback. + +Checker/schema evolution uses a two-release promotion protocol: + +1. **Compatibility release:** merge disabled, backward-compatible parsing and + reporting support while the old trusted checker can still evaluate the tree. + Publish and attest checker `N+1`. +2. **Trust transition:** update protected control-plane/base policy to trust both + `N` and `N+1`. Run both on all candidate and merge-group snapshots; combine + verdicts by the most conservative result. `UNSUPPORTED` is not success. +3. **Behavior activation:** in a later PR, activate the new schema/classification + only after both trusted checkers understand the transition representation. +4. **Retirement:** after the compatibility window and rollback test, remove `N` + from protected policy in another reviewed control change. + +An emergency security checker upgrade uses a pre-authorized break-glass control +outside the candidate branch, requires two protected approvers, emits an immutable +audit record, and may only preserve or strengthen blockers. It cannot waive unit +drift or accept candidate-authored evidence. + +### 8.3 Baseline policy + +Baseline resets require: + +- Exact unit scope. +- Human-readable reason. +- Actor and timestamp. +- Current Git commit or dirty-tree declaration. +- `semantic_status=UNKNOWN` unless validation runs in the same transaction. +- A baseline-reset unit cannot pass protected enforcement while `UNKNOWN` unless a + valid waiver is bound to that exact subject snapshot and manifest digest. +- Review in a normal PR. + +Repository-wide baseline reset is prohibited except for the one audited migration +PR per repository. That PR must contain only baseline/waiver data and its audit +report. + +### 8.4 Waiver policy + +Every waiver contains: + +```json +{ + "subject": {"kind": "unit|candidate", "id": "..."}, + "status": "DRIFTED|UNBASELINED|CORRUPT|UNKNOWN|CONFLICT|FAILED|INVALID", + "owner": "team-or-user", + "approved_by": "protected-reviewer", + "reason": "...", + "issue": "https://...", + "verdict_digest": "...", + "snapshot_digest": "...", + "manifest_digest": "...", + "dependency_closure_digest": "...", + "issued_from_base_sha": "...", + "created_at": "...", + "expires_at": "..." +} +``` + +Expired, malformed, overbroad, and stale waivers fail CI. A waiver suppresses the +merge block only for its exact bound snapshot/verdict. Policy and waiver authority +come from the protected base/control plane, and changes require protected approval. +Reports and nightly telemetry continue to show the condition. + +`issued_from_base_sha` is audit provenance, not an equality condition that expires +on every unrelated merge. Validity is bound to the immutable subject snapshot, +manifest, verdict, and affected dependency closure. Unrelated base movement keeps +the waiver valid; a change in the unit or affected closure invalidates it. Both +cases have merge-group lifecycle tests. + +Waivers never change intrinsic inventory status or reduce the gross managed +denominator. Reports show `managed_total`, `managed_in_sync`, `managed_waived`, and +each non-managed inventory bucket separately. Waivers are an incident mechanism, +not migration debt: epic closure and initial enforcement activation require +`managed_waived=0` in PDD and pdd_cloud. + +### 8.5 Attestation trust policy + +A protected-base/control-plane `AttestationTrustPolicy` defines which evidence can +be trusted. Its digest is part of every effective evidence decision. It specifies: + +- Accepted issuer roots, keyless OIDC issuers/subjects or key IDs, algorithms, and + threshold requirements for protected human-review attestations. +- Exact repository UUID, workflow identity/version, protected workflow source, + environment, ref class, validator/checker release, and permitted claim values. +- Binding to a nonce/check-run ID, obligation/profile digest, complete input and + dependency closure, runner/config digest, base/head/merge-group SHA, result, and + issuance/expiry window to prevent cross-repository or cross-snapshot replay. +- Transparency/audit-log requirements and immutable links from the check result. +- Overlapping key/root rotation, revocation lists, maximum validity, and fail-closed + behavior for unknown, expired, malformed, replayed, or revoked attestations. +- Compromise response: revoke the issuer/key/workflow identity, invalidate affected + cached evidence, rotate through protected policy, and re-run obligations before + any unit can return to `VERIFIED`. + +Signing occurs only in the trusted post-validation service. Sandboxed candidate +code, model jobs, repair jobs, and test runners never receive signing keys/tokens. +Keyless claims are checked against the protected workflow and repository identity, +not merely a CI provider name. + +## 9. Observability and operational ownership + +### 9.1 Stable report fields + +Every command and automation path emits: + +- Report schema version and PDD version. +- Project root and manifest digest. +- Total candidates and every inventory bucket. +- Baseline and semantic status counts. +- Changed/affected artifacts and provenance. +- Recommended and attempted action. +- Transaction ID and finalization result. +- Validation/evidence digest. +- Waiver status. +- Cost, duration, and bounded-retry counts for repair paths. +- Explicit skipped/degraded reasons. + +### 9.2 Metrics and service objectives + +- Inventory coverage: 100% classified. +- Silent omissions: 0. +- Managed units on protected `main`: 100% satisfy the complete trusted `IN_SYNC` + predicate at program completion. Gross managed, managed-in-sync, and + managed-waived counts are reported separately; initial activation and epic + closure require managed-waived and semantic `UNKNOWN` counts of 0. +- Drift dwell time: resolved or surfaced as blocking conflict by the next PR check; + nightly is the fallback and must remain under 24 hours. +- Finalization success for successful mutators: 100%. +- Second-run writes after successful repair: 0. +- False-success incidents: 0. +- Automatic conflict winner selection: 0. +- Candidate-authored evidence accepted as trusted: 0. +- Repair jobs with production secrets or push credentials: 0. +- Hook changed-unit classification target: under 2 seconds for a five-file commit + in the reference repository. +- Full 500-unit read-only reconcile target: recorded and enforced after a baseline + benchmark establishes a realistic threshold. + +### 9.3 Ownership + +Assign one named owner before each workstream starts: + +| Area | Required owner responsibility | +| --- | --- | +| Sync core | identity, graph, hashing, classifier API | +| Persistence | schema, lock, transaction, rollback | +| Repair | planner, sync/update/fix/resolve semantics | +| CI integration | hooks, ledger, PR/merge/nightly checks | +| Evidence trust service | policy loader, signer/verifier, nonce, transparency, human attestations, rotation/revocation | +| PDD dogfood | inventory and baseline burn-down | +| pdd_cloud rollout | consumer audit, canaries, vendored retirement | + +The tracking epic records owner, PR, state, and exit-gate evidence for every row. + +## 10. Failure handling and rollback + +- Core classifier rollout starts in report mode and writes no metadata. +- Transaction rollout retains pre-state snapshots and can be disabled per command. +- CI enforcement can return to report mode without changing stored baselines. +- Durable journals are recovered before any new mutation; a corrupt journal stops + mutation and preserves destination bytes for manual recovery. +- Bot repair never force-pushes and never edits a conflict. +- A failed post-merge/nightly run opens or updates an incident/repair PR; it does + not stamp `main` green. +- Schema migration retains the prior file until atomic replacement succeeds. +- Any increase in false success, data loss, or unit omission is a stop-the-line + rollback condition. +- Any consumer difference between vendored and upstream classification blocks + vendored retirement until explained by an approved behavior change. +- Policy/waiver/checker changes in a candidate PR are evaluated by protected-base + policy and a trusted released checker, so rollback cannot be authorized by the + change under test. + +## 11. Definition of Done + +The global sync epic may close only when all conditions below hold with attached +commands, commit SHAs, and reports. + +1. One canonical unit resolver, include parser, hash builder, classifier, + verification-profile evaluator, fingerprint writer, and command capability + registry remain in production code. +2. `sync_determine_operation.py` is a thin compatibility/workflow adapter and no + longer owns drift classification or persistence. +3. The base/head candidate union includes configured roots, prompts, architecture, + fingerprints, ownership, and unmatched tracked artifacts; every candidate is + managed, human-owned, removed, or invalid in source and installed-wheel + environments; waiver state is reported separately. An independent Git base/head + tree partition has no unaccounted tracked path. + Protected in-repository aliases preserve logical identity without permitting + retarget/escape, and every language-registry row/output role passes exhaustive + source/wheel conformance with no ambiguous mapping. +4. Every successful mutating command finalizes through a crash-durable journal, + ordered shared-resource locks, and compare-and-swap preconditions, and is a + no-op on immediate rerun. +5. `pdd reconcile` is read-only unless an explicit baseline command is invoked. +6. Baseline resets remain semantic `UNKNOWN` until current evidence verifies them. +7. Prompt/code co-edits cannot be auto-healed and retain recoverable ancestry. +8. Prompt, code, docs, examples, and tests follow the canonical affected graph and + ownership model, and every managed read/write satisfies protected `PathPolicy` + including descriptor-relative no-follow commit checks. Content, artifact type, + normalized Git mode, and protected permission bits remain synchronized through + classification, commit, rollback, recovery, Git, and source/wheel execution. +9. Declared runner adapters collect and execute every PDD-owned and human-owned + validation test in `validated_artifacts`; human-owned tests are never + overwritten without an explicit protected adoption transition. + Every managed unit has a complete protected verification profile, and every + required obligation has current trusted evidence; no validator/profile/test + change certifies itself. +10. A trusted released/base checker and protected-base policy classify the current + base/head result and merge-group SHA. Every managed unit must satisfy the full + trusted `IN_SYNC` predicate; any temporary operational exception requires an + active waiver bound to its verdict, snapshot, manifest, affected closure, + issuance provenance, protected approver, and expiry. + Trusted semantic evidence satisfies the protected attestation issuer/subject, + replay, validity, rotation, revocation, and transparency policy. +11. Post-merge and nightly checks are read-only verifiers and converge to no-op. +12. The lifecycle matrix passes in fixture repos, built wheels, PDD staging, and a + pdd_cloud canary. Unresolved issue-linked strict xfails, unapproved required-lane + skips/xfails/deselections/quarantines, collection errors, timeouts, and flaky + obligation outcomes are 0. +13. PDD and pdd_cloud `main` report complete candidate-union inventory, all managed + units trusted `IN_SYNC`, `managed_waived=0`, and zero managed `DRIFTED`, + `UNBASELINED`, `CORRUPT`, `UNKNOWN`, `CONFLICT`, `FAILED`, or `INVALID` states. + Protected expected-managed totals have not been reduced through debt-clearing + ownership transitions. +14. pdd_cloud no longer vendors sync identity, candidate discovery, include + parsing, graph construction, affected closure, path resolution, hashing, or + classification logic; it retains orchestration only. +15. Seven consecutive full nightly runs in both repositories are no-ops except for + deliberately injected canary drift, which is detected and resolved as expected; + ledger/cursor deletion during the window does not narrow the scan. +16. Issue #1932 child issues are closed only with links to the tests and production + paths that satisfy their acceptance criteria. + +## 12. Immediate next actions + +1. Reopen/reframe issue #1932 and attach this document. +2. Mark PR #1954 as the reporting/conflict-preservation milestone. +3. Keep PR #1969 blocked; split inventory, implementation, data baseline, and CI + enforcement into separate PRs. +4. Land PR 0 strict issue-linked expected failures in the nonblocking acceptance + lane before changing the resolver again. +5. Implement PR 1 and PR 2 before any new repository-wide stamp. +6. Complete the shared classifier and transaction foundations before enabling + auto-repair or merge enforcement. +7. Run boundary checks in shadow mode, land the audited data-only PDD baseline, + then activate protected enforcement in a separate operational PR. +8. Maintain the pdd_cloud prompt-side reconciler in report/repair scope until the + released upstream replacement passes parity and canary gates. diff --git a/pdd/commands/__init__.py b/pdd/commands/__init__.py index 21e8725eb..b36d1a27d 100644 --- a/pdd/commands/__init__.py +++ b/pdd/commands/__init__.py @@ -24,6 +24,7 @@ from .firecrawl import firecrawl_cache from .story import story_cli from .reconcile import install_hooks, reconcile +from .sync_core import baseline, certify, recover, validate def register_commands(cli: click.Group) -> None: """Register all subcommands with the main CLI group.""" @@ -55,6 +56,10 @@ def register_commands(cli: click.Group) -> None: cli.add_command(which) cli.add_command(reconcile) cli.add_command(install_hooks) + cli.add_command(certify) + cli.add_command(recover) + cli.add_command(baseline) + cli.add_command(validate) # Register templates group directly to commands dict to handle nesting if needed, # or just add_command works for groups too. diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py new file mode 100644 index 000000000..b37c2ca94 --- /dev/null +++ b/pdd/commands/sync_core.py @@ -0,0 +1,328 @@ +"""Canonical synchronization certification and recovery commands.""" + +from __future__ import annotations + +import json +import os +import subprocess +import uuid +from datetime import datetime, timezone +from pathlib import Path +from pathlib import PurePosixPath +from typing import Optional + +import click + +from ..sync_core import ( + CanonicalReportOptions, + FingerprintProvenance, + FingerprintRecord, + FingerprintStore, + GlobalCertificateOptions, + PlannedWrite, + RepositoryTarget, + SemanticStatus, + TransactionManager, + attestation_signer_from_environment, + build_canonical_report, + build_global_certificate, + build_unit_manifest, + build_unit_snapshot, + checker_identity_from_environment, + encode_fingerprint, + finalize_unit, + load_verification_profiles, + run_lifecycle_matrix, + signer_from_environment, +) +from .. import __version__ + + +def _repository_path(name: str, cwd: Path) -> Path: + """Resolve a named checkout without silently choosing another repository.""" + normalized = name.replace("-", "_") + configured = os.environ.get(f"PDD_CERTIFY_{normalized.upper()}_PATH") + candidates = [Path(configured)] if configured else [] + candidates.extend([cwd, cwd.parent / name, cwd.parent / normalized]) + candidates.extend(parent for parent in cwd.parents if parent.name == name) + candidates.extend(parent for parent in cwd.parents if parent.name == normalized) + for candidate in candidates: + is_named = candidate.name in {name, normalized} + is_pdd_checkout = name == "pdd" and (candidate / "pdd" / "__init__.py").is_file() + if (is_named or is_pdd_checkout) and (candidate / ".git").exists(): + return candidate.resolve() + raise ValueError(f"cannot resolve repository checkout: {name}") + + +def _global_targets( + repositories: str, merge_group: str, cwd: Path +) -> tuple[RepositoryTarget, ...]: + if len(merge_group) != 40 or any( + character not in "0123456789abcdef" for character in merge_group + ): + raise ValueError("--merge-group must be an exact lowercase commit SHA") + names = tuple(item.strip() for item in repositories.split(",") if item.strip()) + if names != ("pdd", "pdd_cloud"): + raise ValueError("--repos must name exactly pdd,pdd_cloud") + targets = [] + for name in names: + path = _repository_path(name, cwd) + if name == "pdd": + head_ref = merge_group + base_ref = os.environ.get( + "PDD_CERTIFY_PDD_BASE_SHA", f"{merge_group}^1" + ) + else: + head_ref = os.environ.get("PDD_CERTIFY_PDD_CLOUD_HEAD_SHA") + base_ref = os.environ.get("PDD_CERTIFY_PDD_CLOUD_BASE_SHA") + if not head_ref or not base_ref: + raise ValueError( + "protected runner must set PDD_CERTIFY_PDD_CLOUD_BASE_SHA " + "and PDD_CERTIFY_PDD_CLOUD_HEAD_SHA" + ) + if ( + len(head_ref) != 40 + or len(base_ref) != 40 + or any(character not in "0123456789abcdef" for character in head_ref + base_ref) + or head_ref == base_ref + ): + raise ValueError( + "protected pdd_cloud base/head must be distinct lowercase commit SHAs" + ) + targets.append(RepositoryTarget(name, path, base_ref, head_ref)) + return tuple(targets) + + +def _head_sha(root: Path) -> str: + result = subprocess.run( + ["git", "rev-parse", "--verify", "HEAD^{commit}"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise ValueError("cannot resolve repository HEAD") + return result.stdout.strip() + + +@click.command("certify") +@click.option("--base-ref", default="HEAD", show_default=True) +@click.option("--head-ref", default="HEAD", show_default=True) +@click.option( + "--replay-ledger", + type=click.Path(path_type=Path), + envvar="PDD_ATTESTATION_REPLAY_LEDGER", +) +@click.option("--module", "modules", multiple=True) +@click.option("--repos", help="Comma-separated repositories for global certification.") +@click.option("--merge-group", help="Exact protected PDD merge-group commit SHA.") +@click.option("--full-inventory", is_flag=True) +@click.option("--run-lifecycle-matrix", "run_matrix", is_flag=True) +@click.option("--require-nightly-streak", type=click.IntRange(min=1)) +@click.option( + "--nightly-ledger", + type=click.Path(path_type=Path, dir_okay=False), + envvar="PDD_NIGHTLY_CERTIFICATE_LEDGER", +) +@click.option("--output", type=click.Path(path_type=Path, dir_okay=False)) +@click.pass_context +def certify( + ctx: click.Context, + **options, +) -> None: + # pylint: disable=too-many-locals + """Produce the strict machine-verifiable canonical sync report.""" + ctx.ensure_object(dict) + ctx.obj["_suppress_result_summary"] = True + base_ref = str(options["base_ref"]) + head_ref = str(options["head_ref"]) + trust_root = Path( + os.environ.get( + "PDD_SYNC_TRUST_ROOT", Path.home() / ".pdd/trust/global-sync" + ) + ) + modules = tuple(str(item) for item in options["modules"]) + output_value = options.get("output") + output: Optional[Path] = Path(output_value) if output_value is not None else None + try: + repositories = options.get("repos") + replay_value = options.get("replay_ledger") + replay_ledger = ( + Path(replay_value) + if replay_value is not None + else trust_root / ("replay" if repositories is not None else "single.json") + ) + if repositories is not None: + required = { + "--merge-group": options.get("merge_group"), + "--full-inventory": options.get("full_inventory"), + "--run-lifecycle-matrix": options.get("run_matrix"), + "--require-nightly-streak": options.get("require_nightly_streak"), + } + missing = [name for name, value in required.items() if not value] + if missing: + raise ValueError("global certification requires " + ", ".join(missing)) + targets = _global_targets( + str(repositories), str(options["merge_group"]), Path.cwd().resolve() + ) + if replay_ledger.exists() and not replay_ledger.is_dir(): + raise ValueError("global --replay-ledger must be a directory") + replay_ledger.mkdir(parents=True, exist_ok=True) + report = build_global_certificate( + targets, + GlobalCertificateOptions( + replay_ledger_root=replay_ledger, + lifecycle_result=run_lifecycle_matrix( + targets[0].path, + cloud_root=targets[1].path, + cloud_base_ref=targets[1].base_ref, + cloud_head_ref=targets[1].head_ref, + ), + nightly_ledger=( + Path(options["nightly_ledger"]) + if options.get("nightly_ledger") is not None + else trust_root / "nightly.jsonl" + ), + required_nightly_streak=int(options["require_nightly_streak"]), + checker_identity=checker_identity_from_environment(), + ), + signer=signer_from_environment(), + ) + else: + report = build_canonical_report( + Path.cwd(), + CanonicalReportOptions( + base_ref=base_ref, + head_ref=head_ref, + modules=modules, + replay_ledger_path=replay_ledger, + ), + ) + except (OSError, RuntimeError, ValueError) as exc: + report = {"schema_version": 1, "ok": False, "errors": [str(exc)]} + encoded = json.dumps(report, indent=2, sort_keys=True) + "\n" + if output is not None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(encoded, encoding="utf-8") + click.echo(encoded, nl=False) + if not report.get("ok"): + raise click.exceptions.Exit(1) + + +@click.command("recover") +@click.option("--transaction", "transaction_id", required=True) +@click.pass_context +def recover(ctx: click.Context, transaction_id: str) -> None: + """Explicitly recover one crash-durable synchronization transaction.""" + ctx.ensure_object(dict) + result = TransactionManager(Path.cwd()).recover(transaction_id) + click.echo( + json.dumps( + { + "transaction_id": result.transaction_id, + "phase": result.phase.value, + "changed_paths": [path.as_posix() for path in result.changed_paths], + "no_op": result.no_op, + }, + indent=2, + sort_keys=True, + ) + ) + + +@click.command("baseline") +@click.option("--module", required=True, help="Exact repository-relative prompt path.") +@click.option("--reviewed-by", required=True) +@click.option("--reason", required=True) +@click.pass_context +def baseline(ctx: click.Context, module: str, reviewed_by: str, reason: str) -> None: + # pylint: disable=too-many-locals + """Record reviewed current bytes as CURRENT plus semantic UNKNOWN.""" + ctx.ensure_object(dict) + root = Path.cwd().resolve() + head = _head_sha(root) + manifest = build_unit_manifest(root, base_ref=head, head_ref=head) + wanted = PurePosixPath(module) + matches = [ + unit for unit in manifest.managed_units if unit.unit_id.prompt_relpath == wanted + ] + if len(matches) != 1: + raise click.ClickException( + f"baseline requires exactly one managed prompt match: {module}" + ) + profiles = load_verification_profiles(root, manifest) + profile = profiles.for_unit(matches[0].unit_id) + if profile is None: + raise click.ClickException("baseline requires a protected verification profile") + snapshot = build_unit_snapshot(root, manifest, matches[0], profile) + transaction_id = f"baseline-{uuid.uuid4()}" + provenance = FingerprintProvenance( + "baseline-reset", + f"pdd baseline --module {module}", + transaction_id, + head, + datetime.now(timezone.utc).isoformat(), + __version__, + reviewed_by.strip(), + reason.strip(), + ) + record = FingerprintRecord( + snapshot, 2, 2, provenance, SemanticStatus.UNKNOWN, None + ) + store = FingerprintStore(root) + store.validate(record) + relpath = store.path_for(record.snapshot.unit_id).relative_to(root) + manager = TransactionManager(root) + manager.prepare( + transaction_id, + ( + PlannedWrite( + PurePosixPath(relpath.as_posix()), + encode_fingerprint(record), + "100644", + ), + ), + ) + result = manager.commit(transaction_id) + click.echo( + json.dumps( + { + "transaction_id": result.transaction_id, + "baseline": "CURRENT", + "semantic": "UNKNOWN", + "fingerprint": relpath.as_posix(), + }, + indent=2, + sort_keys=True, + ) + ) + + +@click.command("validate") +@click.option("--module", required=True, help="Exact repository-relative prompt path.") +@click.option("--base-ref", required=True, help="Protected base commit or ref.") +@click.option("--head-ref", default="HEAD", show_default=True) +@click.pass_context +def validate(ctx: click.Context, module: str, base_ref: str, head_ref: str) -> None: + """Run protected obligations and transactionally finalize trusted evidence.""" + ctx.ensure_object(dict) + result = finalize_unit( + Path.cwd(), + PurePosixPath(module), + base_ref=base_ref, + head_ref=head_ref, + signer=attestation_signer_from_environment(), + ) + click.echo( + json.dumps( + { + "transaction_id": result.transaction.transaction_id, + "attestation_id": result.attestation_id, + "fingerprint": result.fingerprint_path.as_posix(), + "semantic": "VERIFIED", + }, + indent=2, + sort_keys=True, + ) + ) diff --git a/pdd/core/cli.py b/pdd/core/cli.py index b387b684f..67ef1994e 100644 --- a/pdd/core/cli.py +++ b/pdd/core/cli.py @@ -350,6 +350,12 @@ def _write_result_core_dump( class PDDCLI(click.Group): """Custom Click Group that adds a Generate Suite section to root help.""" + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + """Reserve ``sync certify`` as the global certificate command spelling.""" + if len(args) >= 2 and args[0:2] == ["sync", "certify"]: + args = ["certify", *args[2:]] + return super().parse_args(ctx, args) + def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: self.format_usage(ctx, formatter) formatter.write_text(PDD_FULL_TAGLINE) diff --git a/pdd/sync_core/__init__.py b/pdd/sync_core/__init__.py new file mode 100644 index 000000000..2c02990ad --- /dev/null +++ b/pdd/sync_core/__init__.py @@ -0,0 +1,235 @@ +"""Canonical, side-effect-free primitives for global synchronization.""" + +from .classifier import classify +from .capabilities import CapabilityError, CapabilityRegistry, CommandCapability +from .certificate import ( + CheckerIdentity, + GlobalCertificateOptions, + LifecycleResult, + RepositoryTarget, + build_global_certificate, + count_vendored_sync_semantics, + checker_identity_from_environment, + signer_from_environment, + verify_global_certificate, +) +from .identity import ( + RepositoryIdentity, + RepositoryIdentityError, + canonical_repository_id, + initialize_repository_identity, + read_repository_identity, +) +from .fingerprint_store import ( + CorruptFingerprintError, + FingerprintStore, + FingerprintStoreError, + LegacyFingerprintRecord, + encode_fingerprint, +) +from .finalize import ( + CanonicalFinalizationError, + FinalizeResult, + attestation_signer_from_environment, + canonical_root_for_paths, + finalize_legacy_paths, + finalize_unit, +) +from .evidence_store import ( + EvidenceStoreError, + LoadedTrustPolicy, + decode_attestation, + encode_attestation, + evidence_relpath, + load_attestation, + load_trust_policy, +) +from .includes import ( + IncludeClosure, + IncludeEdge, + IncludeGraphError, + IncludeReference, + IncludeSyntax, + build_include_closure, + include_paths, + parse_include_references, +) +from .language import LanguageRegistry, LanguageRegistryError, LanguageSpec +from .lifecycle import run_lifecycle_matrix +from .manifest import ( + CandidateRecord, + ManifestError, + ManifestUnit, + UnitManifest, + build_unit_manifest, +) +from .path_policy import PathPolicy, PathPolicyError, ResolvedPath +from .planner import RepairAction, RepairPlan, RepairPolicy, plan_repair +from .snapshot import SnapshotError, build_unit_snapshot +from .reporting import ( + CanonicalCounts, + CanonicalReportOptions, + build_canonical_report, +) +from .runner import ( + AttestationIssue, + RunBinding, + RunnerConfig, + RunnerExecution, + TRUSTED_RUNNER_VERSION, + run_obligation, + run_profile, + runner_identity_digest, +) +from .trust import ( + AttestationBinding, + AttestationEnvelope, + AttestationError, + AttestationRequest, + AttestationSigner, + AttestationTrustPolicy, + FileReplayStore, + InMemoryReplayStore, + ValidationEvidence, +) +from .transaction import ( + PlannedWrite, + RecoveryRequired, + TransactionConflict, + TransactionError, + TransactionManager, + TransactionPhase, + TransactionResult, +) +from .verification import ( + ProfileSet, + VerificationProfileError, + load_verification_profiles, +) +from .waivers import SyncWaiver, WaiverSet, load_sync_waivers +from .types import ( + ArtifactSnapshot, + BaselineStatus, + CandidateId, + EvidenceOutcome, + FingerprintRecord, + FingerprintProvenance, + InventoryStatus, + SemanticStatus, + SyncVerdict, + UnitId, + UnitSnapshot, + VerificationObligation, + VerificationProfile, +) + +__all__ = [ + "ArtifactSnapshot", + "AttestationBinding", + "AttestationEnvelope", + "AttestationError", + "AttestationIssue", + "AttestationRequest", + "AttestationSigner", + "AttestationTrustPolicy", + "BaselineStatus", + "CandidateId", + "CandidateRecord", + "CanonicalCounts", + "CanonicalFinalizationError", + "CanonicalReportOptions", + "CapabilityError", + "CapabilityRegistry", + "CommandCapability", + "CorruptFingerprintError", + "EvidenceOutcome", + "EvidenceStoreError", + "FingerprintRecord", + "FingerprintProvenance", + "FingerprintStore", + "FingerprintStoreError", + "FileReplayStore", + "FinalizeResult", + "GlobalCertificateOptions", + "IncludeReference", + "IncludeSyntax", + "IncludeClosure", + "IncludeEdge", + "IncludeGraphError", + "InventoryStatus", + "InMemoryReplayStore", + "LanguageRegistry", + "LanguageRegistryError", + "LanguageSpec", + "LegacyFingerprintRecord", + "LoadedTrustPolicy", + "LifecycleResult", + "ManifestError", + "ManifestUnit", + "PathPolicy", + "PathPolicyError", + "PlannedWrite", + "ProfileSet", + "RecoveryRequired", + "RepairAction", + "RepairPlan", + "RepairPolicy", + "RunnerConfig", + "RunnerExecution", + "RunBinding", + "RepositoryIdentity", + "RepositoryIdentityError", + "RepositoryTarget", + "ResolvedPath", + "SemanticStatus", + "SnapshotError", + "SyncVerdict", + "SyncWaiver", + "TransactionConflict", + "TransactionError", + "TransactionManager", + "TransactionPhase", + "TransactionResult", + "TRUSTED_RUNNER_VERSION", + "UnitId", + "UnitManifest", + "UnitSnapshot", + "ValidationEvidence", + "VerificationObligation", + "VerificationProfile", + "VerificationProfileError", + "WaiverSet", + "classify", + "canonical_repository_id", + "canonical_root_for_paths", + "decode_attestation", + "encode_attestation", + "encode_fingerprint", + "evidence_relpath", + "finalize_unit", + "finalize_legacy_paths", + "build_include_closure", + "build_canonical_report", + "build_global_certificate", + "CheckerIdentity", + "checker_identity_from_environment", + "build_unit_manifest", + "build_unit_snapshot", + "attestation_signer_from_environment", + "count_vendored_sync_semantics", + "include_paths", + "load_verification_profiles", + "load_attestation", + "load_trust_policy", + "load_sync_waivers", + "initialize_repository_identity", + "read_repository_identity", + "run_obligation", + "run_lifecycle_matrix", + "run_profile", + "runner_identity_digest", + "signer_from_environment", + "parse_include_references", + "plan_repair", + "verify_global_certificate", +] diff --git a/pdd/sync_core/capabilities.py b/pdd/sync_core/capabilities.py new file mode 100644 index 000000000..42da75238 --- /dev/null +++ b/pdd/sync_core/capabilities.py @@ -0,0 +1,127 @@ +"""Protected command mutation scopes and transaction requirements.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Iterable + + +class CapabilityError(ValueError): + """Raised when a command attempts an undeclared synchronization mutation.""" + + +@dataclass(frozen=True, order=True) +class CommandCapability: + """Protected read/write contract for one command or automation consumer.""" + + command_id: str + readable_roles: frozenset[str] + writable_roles: frozenset[str] + shared_resources: tuple[PurePosixPath, ...] + requires_transaction: bool + may_reset_baseline: bool = False + may_issue_evidence: bool = False + + @property + def read_only(self) -> bool: + """Return whether the command is prohibited from all managed writes.""" + return not self.writable_roles + + +class CapabilityRegistry: + """Immutable lookup and enforcement for protected command capabilities.""" + + def __init__(self, capabilities: Iterable[CommandCapability]) -> None: + entries = tuple(sorted(capabilities)) + self._entries = {entry.command_id: entry for entry in entries} + if len(self._entries) != len(entries): + raise CapabilityError("duplicate command capability ID") + + @classmethod + def protected_defaults(cls) -> "CapabilityRegistry": + """Return the canonical command scopes used by CLI and automation.""" + all_roles = frozenset( + {"prompt", "include", "code", "example", "test", "fingerprint", "evidence"} + ) + return cls( + ( + CommandCapability("reconcile", all_roles, frozenset(), (), False), + CommandCapability("certify", all_roles, frozenset(), (), False), + CommandCapability( + "baseline", + all_roles, + frozenset({"fingerprint"}), + (), + True, + may_reset_baseline=True, + ), + CommandCapability( + "sync", + all_roles, + frozenset({"code", "example", "test", "fingerprint", "evidence"}), + (PurePosixPath("architecture.json"),), + True, + ), + CommandCapability( + "update", + all_roles, + frozenset({"prompt", "fingerprint", "evidence"}), + (PurePosixPath("architecture.json"),), + True, + ), + CommandCapability( + "resolve", + all_roles, + all_roles - {"include"}, + (PurePosixPath("architecture.json"),), + True, + ), + CommandCapability( + "trusted-validator", + all_roles, + frozenset({"evidence"}), + (), + True, + may_issue_evidence=True, + ), + ) + ) + + def get(self, command_id: str) -> CommandCapability: + """Return one protected capability or fail closed for unknown commands.""" + try: + return self._entries[command_id] + except KeyError as exc: + raise CapabilityError(f"unknown synchronization command: {command_id}") from exc + + def authorize_writes(self, command_id: str, roles: Iterable[str]) -> None: + """Reject writes outside a command's protected role scope.""" + capability = self.get(command_id) + requested = frozenset(roles) + unauthorized = requested - capability.writable_roles + if unauthorized: + raise CapabilityError( + f"{command_id} cannot write roles: {', '.join(sorted(unauthorized))}" + ) + if requested and not capability.requires_transaction: + raise CapabilityError(f"{command_id} has no transactional mutation scope") + + def digest(self) -> str: + """Return a deterministic digest bound into manifests and evidence.""" + payload = [ + { + "command_id": item.command_id, + "readable_roles": sorted(item.readable_roles), + "writable_roles": sorted(item.writable_roles), + "shared_resources": [path.as_posix() for path in item.shared_resources], + "requires_transaction": item.requires_transaction, + "may_reset_baseline": item.may_reset_baseline, + "may_issue_evidence": item.may_issue_evidence, + } + for item in sorted(self._entries.values()) + ] + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py new file mode 100644 index 000000000..37fdca7cc --- /dev/null +++ b/pdd/sync_core/certificate.py @@ -0,0 +1,825 @@ +"""Cross-repository signed Global Sync Certificate aggregation.""" + +from __future__ import annotations + +import base64 +import ast +import json +import os +import subprocess +import fnmatch +import tempfile +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path, PurePosixPath +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from .reporting import CanonicalReportOptions, build_canonical_report +from .trust import AttestationSigner + + +@dataclass(frozen=True) +class RepositoryTarget: + """Repository path and exact protected refs to certify.""" + + name: str + path: Path + base_ref: str = "HEAD" + head_ref: str = "HEAD" + + +@dataclass(frozen=True) +class LifecycleResult: + """Externally produced required-scenario and test-outcome totals.""" + + failed: int + required_tests_skipped_or_xfailed: int + collection_errors: int + timeouts: int + post_repair_second_run_writes: int + post_merge_tree_changes: int + missing_scenarios: tuple[str, ...] = () + + +@dataclass(frozen=True) +class CheckerIdentity: + """Protected released-checker artifact and workflow provenance.""" + + wheel_sha256: str + release_ref: str + workflow_identity: str + + def __post_init__(self) -> None: + if ( + len(self.wheel_sha256) != 64 + or any(character not in "0123456789abcdef" for character in self.wheel_sha256) + or not self.release_ref.startswith("refs/tags/") + or not self.workflow_identity + ): + raise ValueError("released checker identity is malformed") + + def payload(self) -> dict[str, str]: + """Return the canonical signed checker identity payload.""" + return { + "wheel_sha256": self.wheel_sha256, + "release_ref": self.release_ref, + "workflow_identity": self.workflow_identity, + } + + +@dataclass(frozen=True) +class GlobalCertificateOptions: + """Trust and external evidence inputs for global certification.""" + + replay_ledger_root: Path + lifecycle_result: LifecycleResult + nightly_ledger: Path + required_nightly_streak: int = 7 + checker_identity: CheckerIdentity | None = None + + +def _canonical_bytes(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + + +def count_vendored_sync_semantics( + root: Path, *, base_ref: str = "HEAD", head_ref: str = "HEAD" +) -> int: + # pylint: disable=too-many-locals,too-many-boolean-expressions + """Enforce the protected consumer dependency boundary at exact Git refs.""" + repository_root = Path(root).resolve() + try: + raw_policy = _git( + repository_root, + "show", + f"{base_ref}:.pdd/sync-consumer-boundary.json", + ) + policy = json.loads(raw_policy) + patterns = policy["forbidden_paths"] + forbidden_imports = policy["forbidden_imports"] + forbidden_symbols = policy["forbidden_symbols"] + excluded_paths = policy["excluded_paths"] + if ( + policy.get("schema_version") != 1 + or policy.get("canonical_package") != "pdd.sync_core" + or not isinstance(patterns, list) + or not patterns + or not all(isinstance(item, str) and item for item in patterns) + or not isinstance(forbidden_imports, list) + or not all(isinstance(item, str) and item for item in forbidden_imports) + or not isinstance(forbidden_symbols, list) + or not all(isinstance(item, str) and item for item in forbidden_symbols) + or not isinstance(excluded_paths, list) + or not all(isinstance(item, str) and item for item in excluded_paths) + ): + return 1 + tracked = _git(repository_root, "ls-tree", "-r", "--name-only", head_ref) + except (KeyError, json.JSONDecodeError, ValueError): + return 1 + paths = [item for item in tracked.splitlines() if item] + analyzed_paths = [ + path + for path in paths + if not any(fnmatch.fnmatchcase(path, pattern) for pattern in excluded_paths) + ] + count = sum( + any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) + for path in analyzed_paths + ) + excluded = {"node_modules", "private-pdd", "experiments", "research"} + blocked_tokens = tuple(forbidden_imports) + tuple(forbidden_symbols) + declarations = _forbidden_declaration_paths( + repository_root, head_ref, blocked_tokens + ) + count += sum( + path in declarations + and not any(part in excluded for part in PurePosixPath(path).parts) + for path in analyzed_paths + ) + for path in analyzed_paths: + relpath = PurePosixPath(path) + if any(part in excluded for part in relpath.parts): + continue + if relpath.suffix != ".py": + continue + try: + tree = ast.parse(_git(repository_root, "show", f"{head_ref}:{path}")) + except (SyntaxError, ValueError): + count += 1 + continue + for node in ast.walk(tree): + imported = None + if isinstance(node, ast.ImportFrom): + imported = node.module + elif isinstance(node, ast.Import): + imported_names = [item.name for item in node.names] + count += sum( + any( + name == blocked or name.startswith(blocked + ".") + for blocked in forbidden_imports + ) + for name in imported_names + ) + if imported is not None and any( + imported == blocked or imported.startswith(blocked + ".") + for blocked in forbidden_imports + ): + count += 1 + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and any( + symbol.casefold() in node.name.casefold() + for symbol in forbidden_symbols + ): + count += 1 + return count + + +def _forbidden_declaration_paths( + root: Path, ref: str, blocked_tokens: tuple[str, ...] +) -> set[str]: + """Find prompt and architecture declarations containing protected symbols.""" + command = ["git", "grep", "-I", "-l", "-F"] + for token in blocked_tokens: + command.extend(("-e", token)) + command.extend((ref, "--", "*.prompt", "architecture.json", "**/architecture.json")) + result = subprocess.run( + command, cwd=root, capture_output=True, text=True, check=False + ) + if result.returncode not in {0, 1}: + raise ValueError(result.stderr.strip() or "Git declaration scan failed") + return { + line.split(":", 1)[1] if ":" in line else line + for line in result.stdout.splitlines() + if line + } + + +def _git(root: Path, *args: str) -> str: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + raise ValueError(result.stderr.strip() or "Git certification check failed") + return result.stdout.strip() + + +def _validate_target(target: RepositoryTarget) -> RepositoryTarget: + path = Path(target.path).resolve() + base_sha = _git(path, "rev-parse", "--verify", f"{target.base_ref}^{{commit}}") + head_sha = _git(path, "rev-parse", "--verify", f"{target.head_ref}^{{commit}}") + if base_sha == head_sha: + raise ValueError(f"{target.name}: protected base and head must be distinct") + ancestry = subprocess.run( + ["git", "merge-base", "--is-ancestor", base_sha, head_sha], + cwd=path, + capture_output=True, + check=False, + ) + if ancestry.returncode != 0: + raise ValueError(f"{target.name}: protected base is not an ancestor of head") + if _git(path, "rev-parse", "HEAD") != head_sha: + raise ValueError(f"{target.name}: checkout HEAD is not the certified SHA") + if _git(path, "status", "--porcelain", "--untracked-files=all"): + raise ValueError(f"{target.name}: certification requires a clean checkout") + return RepositoryTarget(target.name, path, base_sha, head_sha) + + +def _certification_targets( + targets: tuple[RepositoryTarget, ...], directory: Path +) -> tuple[RepositoryTarget, ...]: + """Create independent exact-SHA clones for all certificate reads.""" + isolated: list[RepositoryTarget] = [] + for target in targets: + destination = directory / target.name + command = [ + "git", + "clone", + "-q", + "--no-local", + "--no-checkout", + str(target.path), + str(destination), + ] + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise ValueError(f"{target.name}: cannot create certification clone") + _git(destination, "checkout", "-q", "--detach", target.head_ref) + _git(destination, "cat-file", "-e", f"{target.base_ref}^{{commit}}") + isolated.append( + RepositoryTarget( + target.name, + destination, + target.base_ref, + target.head_ref, + ) + ) + return tuple(isolated) + + +def _nightly_lineage( + row: dict[str, Any], targets: tuple[RepositoryTarget, ...] +) -> bool: + reports = row.get("repositories") + if not isinstance(reports, list): + return False + by_name = { + item.get("repository"): item for item in reports if isinstance(item, dict) + } + for target in targets: + report = by_name.get(target.name) + if not isinstance(report, dict): + return False + historical_head = report.get("head_sha") + if ( + report.get("repository_id") is None + or not isinstance(historical_head, str) + or len(historical_head) != 40 + ): + return False + current_identity = _git(target.path, "show", f"{target.head_ref}:.pdd/repository-id") + if report["repository_id"] != current_identity: + return False + ancestry = subprocess.run( + ["git", "merge-base", "--is-ancestor", historical_head, target.head_ref], + cwd=target.path, + capture_output=True, + check=False, + ) + if ancestry.returncode != 0: + return False + return True + + +def _complete_nightly( + row: Any, + public_key: bytes, + expected_issuer: str, + targets: tuple[RepositoryTarget, ...], + checker_identity: CheckerIdentity, +) -> bool: + if not isinstance(row, dict) or row.get("scan_ok") is not True: + return False + repositories = row.get("repositories") + counts = row.get("counts") + lifecycle = row.get("lifecycle") + if not isinstance(repositories, list) or { + item.get("repository") for item in repositories if isinstance(item, dict) + } != {"pdd", "pdd_cloud"}: + return False + if not isinstance(counts, dict) or not isinstance(lifecycle, dict): + return False + required_counts = { + "unaccounted_tracked_paths", + "managed_units", + "trusted_in_sync", + "nightly_streak", + } + required_lifecycle = { + "lifecycle_matrix_failed", + "required_tests_skipped_or_xfailed", + "collection_errors", + "timeouts", + } + return ( + required_counts <= counts.keys() + and required_lifecycle <= lifecycle.keys() + and row.get("checker") == checker_identity.payload() + and _nightly_lineage(row, targets) + and _verify_certificate_integrity( + row, public_key, expected_issuer=expected_issuer + ) + ) + + +def _nightly_streak( + path: Path, + public_key: bytes, + expected_issuer: str, + targets: tuple[RepositoryTarget, ...], + checker_identity: CheckerIdentity, +) -> int: + if not path.exists(): + return 0 + if path.is_symlink() or not path.is_file(): + raise ValueError("nightly certificate ledger is unsafe") + try: + rows = [json.loads(line) for line in path.read_text().splitlines() if line] + except json.JSONDecodeError as exc: + raise ValueError("nightly certificate ledger is malformed") from exc + streak = 0 + previous_date = None + today = datetime.now(timezone.utc).date() + for row in reversed(rows): + if not _complete_nightly( + row, public_key, expected_issuer, targets, checker_identity + ): + break + try: + checked_at = datetime.fromisoformat(str(row["checked_at"])) + except (KeyError, ValueError): + break + if checked_at.tzinfo is None: + break + date = checked_at.astimezone(timezone.utc).date() + if previous_date is None and date not in {today, today - timedelta(days=1)}: + break + if previous_date is not None and (previous_date - date).days != 1: + break + streak += 1 + previous_date = date + return streak + + +def _aggregate_counts(reports: list[dict[str, Any]]) -> dict[str, int]: + names = { + "unaccounted_tracked_paths", + "managed_units", + "protected_expected_managed_units", + "managed_waivers", + "DRIFTED", + "UNBASELINED", + "CORRUPT", + "UNKNOWN", + "CONFLICT", + "FAILED", + "INVALID", + "trusted_in_sync", + "verification_profile_complete", + "trusted_current_evidence", + } + result = {name: 0 for name in names} + mapping = { + "DRIFTED": "drifted", + "UNBASELINED": "unbaselined", + "CORRUPT": "corrupt", + "UNKNOWN": "unknown", + "CONFLICT": "conflict", + "FAILED": "failed", + "INVALID": "invalid", + } + for report in reports: + counts = report.get("counts", {}) + for name in names: + result[name] += int(counts.get(mapping.get(name, name), 0)) + return result + + +def _scan_predicate( + counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int] +) -> bool: + managed = counts["managed_units"] + baseline_failures = ("DRIFTED", "UNBASELINED", "CORRUPT") + semantic_failures = ("UNKNOWN", "CONFLICT", "FAILED", "INVALID") + return ( + counts["unaccounted_tracked_paths"] == 0 + and managed > 0 + and managed == counts["protected_expected_managed_units"] + and counts["managed_waivers"] == 0 + and counts["trusted_in_sync"] == managed + and counts["verification_profile_complete"] == managed + and counts["trusted_current_evidence"] == managed + and all(counts[name] == 0 for name in baseline_failures) + and all(counts[name] == 0 for name in semantic_failures) + and lifecycle.failed == 0 + and lifecycle.required_tests_skipped_or_xfailed == 0 + and lifecycle.collection_errors == 0 + and lifecycle.timeouts == 0 + and not lifecycle.missing_scenarios + and extra["pdd_cloud_vendored_sync_semantics"] == 0 + and lifecycle.post_repair_second_run_writes == 0 + and lifecycle.post_merge_tree_changes == 0 + ) + + +def _predicate( + counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int] +) -> bool: + return _scan_predicate(counts, lifecycle, extra) and ( + extra["nightly_streak"] >= extra["required_nightly_streak"] + ) + + +def _canonical_report_predicate(report: dict[str, Any]) -> bool: + counts = report.get("counts") + if not isinstance(counts, dict): + return False + required = { + "unaccounted_tracked_paths", + "managed_units", + "protected_expected_managed_units", + "managed_waivers", + "trusted_in_sync", + "verification_profile_complete", + "trusted_current_evidence", + "drifted", + "unbaselined", + "corrupt", + "unknown", + "conflict", + "failed", + "invalid", + } + if not required <= counts.keys() or any( + not isinstance(counts[name], int) or isinstance(counts[name], bool) + or counts[name] < 0 + for name in required + ): + return False + managed = counts["managed_units"] + return ( + report.get("errors") == [] + and report.get("recovery_required") == [] + and isinstance(report.get("repository_id"), str) + and bool(report["repository_id"]) + and isinstance(report.get("base_sha"), str) + and len(report["base_sha"]) == 40 + and isinstance(report.get("head_sha"), str) + and len(report["head_sha"]) == 40 + and managed > 0 + and counts["unaccounted_tracked_paths"] == 0 + and managed == counts["protected_expected_managed_units"] + and counts["managed_waivers"] == 0 + and counts["trusted_in_sync"] == managed + and counts["verification_profile_complete"] == managed + and counts["trusted_current_evidence"] == managed + and all( + counts[name] == 0 + for name in ( + "drifted", + "unbaselined", + "corrupt", + "unknown", + "conflict", + "failed", + "invalid", + ) + ) + ) + + +def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool]: + # pylint: disable=too-many-return-statements + if body.get("schema_version") != 2: + return False, False + checker = body.get("checker") + try: + if not isinstance(checker, dict): + return False, False + CheckerIdentity( + str(checker["wheel_sha256"]), + str(checker["release_ref"]), + str(checker["workflow_identity"]), + ) + except (KeyError, ValueError): + return False, False + reports = body.get("repositories") + counts_payload = body.get("counts") + lifecycle_payload = body.get("lifecycle") + if ( + not isinstance(reports, list) + or not isinstance(counts_payload, dict) + or not isinstance(lifecycle_payload, dict) + ): + return False, False + if {item.get("repository") for item in reports if isinstance(item, dict)} != { + "pdd", + "pdd_cloud", + } or len(reports) != 2: + return False, False + report_results = [ + _canonical_report_predicate(item) if isinstance(item, dict) else False + for item in reports + ] + if any( + not isinstance(item, dict) or item.get("ok") is not result + for item, result in zip(reports, report_results) + ): + return False, False + aggregate = _aggregate_counts(reports) + if any(counts_payload.get(name) != value for name, value in aggregate.items()): + return False, False + lifecycle_names = { + "lifecycle_matrix_failed": "failed", + "required_tests_skipped_or_xfailed": "required_tests_skipped_or_xfailed", + "collection_errors": "collection_errors", + "timeouts": "timeouts", + "post_repair_second_run_writes": "post_repair_second_run_writes", + "post_merge_tree_changes": "post_merge_tree_changes", + } + if any( + not isinstance(lifecycle_payload.get(name), int) + or isinstance(lifecycle_payload.get(name), bool) + or lifecycle_payload[name] < 0 + for name in lifecycle_names + ) or not isinstance(lifecycle_payload.get("missing_scenarios"), list): + return False, False + lifecycle = LifecycleResult( + *(lifecycle_payload[name] for name in lifecycle_names), + tuple(lifecycle_payload["missing_scenarios"]), + ) + extra_names = { + "pdd_cloud_vendored_sync_semantics", + "nightly_streak", + "required_nightly_streak", + } + if any( + not isinstance(counts_payload.get(name), int) + or isinstance(counts_payload.get(name), bool) + or counts_payload[name] < 0 + for name in extra_names + ): + return False, False + extra = {name: counts_payload[name] for name in extra_names} + expected_profile_coverage = ( + 100 + if aggregate["managed_units"] > 0 + and aggregate["verification_profile_complete"] == aggregate["managed_units"] + else 0 + ) + expected_evidence_coverage = ( + 100 + if aggregate["managed_units"] > 0 + and aggregate["trusted_current_evidence"] == aggregate["managed_units"] + else 0 + ) + if ( + counts_payload.get("verification_profile_coverage") + != expected_profile_coverage + or counts_payload.get("trusted_current_evidence_coverage") + != expected_evidence_coverage + ): + return False, False + scan_ok = all(report_results) and _scan_predicate(aggregate, lifecycle, extra) + ok = all(report_results) and _predicate(aggregate, lifecycle, extra) + return scan_ok, ok + + +def _build_global_certificate_from_targets( + targets: tuple[RepositoryTarget, ...], + options: GlobalCertificateOptions, + *, + signer: AttestationSigner, +) -> dict[str, Any]: + """Build and sign the complete cross-repository machine predicate.""" + if not targets: + raise ValueError("global certificate requires at least one repository") + if options.checker_identity is None: + raise ValueError("global certificate requires released checker identity") + reports = [] + for target in targets: + report = build_canonical_report( + target.path, + CanonicalReportOptions( + base_ref=target.base_ref, + head_ref=target.head_ref, + replay_ledger_path=options.replay_ledger_root / f"{target.name}.json", + ), + ) + reports.append({**report, "repository": target.name}) + counts = _aggregate_counts(reports) + cloud = next((target for target in targets if target.name == "pdd_cloud"), None) + extra = { + "pdd_cloud_vendored_sync_semantics": ( + count_vendored_sync_semantics( + cloud.path, base_ref=cloud.base_ref, head_ref=cloud.head_ref + ) + if cloud + else 1 + ), + "nightly_streak": _nightly_streak( + options.nightly_ledger, + signer.public_key_bytes(), + signer.issuer, + targets, + options.checker_identity, + ), + "required_nightly_streak": options.required_nightly_streak, + } + lifecycle = options.lifecycle_result + body: dict[str, Any] = { + "schema_version": 2, + "checked_at": datetime.now(timezone.utc).isoformat(), + "checker": options.checker_identity.payload(), + "repositories": reports, + "counts": { + **counts, + **extra, + "verification_profile_coverage": ( + 100 + if counts["managed_units"] > 0 + and counts["verification_profile_complete"] == counts["managed_units"] + else 0 + ), + "trusted_current_evidence_coverage": ( + 100 + if counts["managed_units"] > 0 + and counts["trusted_current_evidence"] == counts["managed_units"] + else 0 + ), + }, + "lifecycle": { + "lifecycle_matrix_failed": lifecycle.failed, + "required_tests_skipped_or_xfailed": lifecycle.required_tests_skipped_or_xfailed, + "collection_errors": lifecycle.collection_errors, + "timeouts": lifecycle.timeouts, + "post_repair_second_run_writes": lifecycle.post_repair_second_run_writes, + "post_merge_tree_changes": lifecycle.post_merge_tree_changes, + "missing_scenarios": list(lifecycle.missing_scenarios), + }, + } + body["scan_ok"] = all(report.get("ok") for report in reports) and _scan_predicate( + counts, lifecycle, extra + ) + body["ok"] = all(report.get("ok") for report in reports) and _predicate( + counts, lifecycle, extra + ) + return { + **body, + "signature": { + "algorithm": "Ed25519", + "issuer": signer.issuer, + "value": signer.sign_bytes(_canonical_bytes(body)), + }, + } + + +def build_global_certificate( + targets: tuple[RepositoryTarget, ...], + options: GlobalCertificateOptions, + *, + signer: AttestationSigner, +) -> dict[str, Any]: + """Build from independent exact-SHA clones and revalidate before signing.""" + if not targets: + raise ValueError("global certificate requires at least one repository") + validated = tuple(_validate_target(target) for target in targets) + with tempfile.TemporaryDirectory(prefix="pdd-global-certificate-") as directory: + isolated = _certification_targets(validated, Path(directory)) + certificate = _build_global_certificate_from_targets( + isolated, options, signer=signer + ) + revalidated = tuple(_validate_target(target) for target in isolated) + if tuple((item.base_ref, item.head_ref) for item in revalidated) != tuple( + (item.base_ref, item.head_ref) for item in isolated + ): + raise ValueError("certification clone refs changed during verification") + return certificate + + +def _verify_certificate_integrity( + certificate: dict[str, Any], + public_key: bytes, + *, + expected_issuer: str, +) -> bool: + """Verify one signed body without treating a red historical scan as accepted.""" + signature = certificate.get("signature") + if ( + not isinstance(signature, dict) + or signature.get("algorithm") != "Ed25519" + or signature.get("issuer") != expected_issuer + ): + return False + body = {key: value for key, value in certificate.items() if key != "signature"} + try: + raw = base64.b64decode(str(signature["value"]), validate=True) + Ed25519PublicKey.from_public_bytes(public_key).verify(raw, _canonical_bytes(body)) + except (KeyError, ValueError, InvalidSignature): + return False + scan_ok, ok = _recompute_certificate_predicates(body) + return body.get("scan_ok") is scan_ok and body.get("ok") is ok + + +def verify_global_certificate( + certificate: dict[str, Any], + public_key: bytes, + *, + expected_issuer: str, + expected_repository_shas: dict[str, tuple[str, str]], + expected_repository_ids: dict[str, str], + expected_checker_identity: CheckerIdentity, + now: datetime | None = None, + maximum_age: timedelta = timedelta(minutes=15), +) -> bool: + # pylint: disable=too-many-arguments,too-many-return-statements + """Accept only a fresh green certificate for exact expected repository refs.""" + if not _verify_certificate_integrity( + certificate, public_key, expected_issuer=expected_issuer + ): + return False + if certificate.get("ok") is not True or certificate.get("scan_ok") is not True: + return False + if certificate.get("checker") != expected_checker_identity.payload(): + return False + try: + checked_at = datetime.fromisoformat(str(certificate["checked_at"])) + except (KeyError, ValueError): + return False + if checked_at.tzinfo is None: + return False + current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + checked_at = checked_at.astimezone(timezone.utc) + if checked_at > current or current - checked_at > maximum_age: + return False + reports = certificate.get("repositories") + if not isinstance(reports, list) or len(reports) != len(expected_repository_shas): + return False + actual = { + str(item.get("repository")): ( + item.get("base_sha"), + item.get("head_sha"), + ) + for item in reports + if isinstance(item, dict) + } + actual_ids = { + str(item.get("repository")): item.get("repository_id") + for item in reports + if isinstance(item, dict) + } + return ( + actual == expected_repository_shas + and actual_ids == expected_repository_ids + and set(expected_repository_shas) == set(expected_repository_ids) + ) + + +def signer_from_environment() -> AttestationSigner: + """Load certificate signing identity from protected runner environment.""" + encoded = os.environ.get("PDD_CERTIFICATE_SIGNING_KEY") + issuer = os.environ.get("PDD_CERTIFICATE_ISSUER", "global-sync-checker") + if not encoded: + raise ValueError("PDD_CERTIFICATE_SIGNING_KEY is required") + try: + key = base64.b64decode(encoded, validate=True) + except ValueError as exc: + raise ValueError("PDD_CERTIFICATE_SIGNING_KEY is malformed") from exc + return AttestationSigner(issuer, key) + + +def checker_identity_from_environment( + *, require_execution_marker: bool = True +) -> CheckerIdentity: + """Load released artifact provenance supplied only by protected CI.""" + if ( + require_execution_marker + and os.environ.get("PDD_RELEASED_CHECKER_EXECUTION") != "1" + ): + raise ValueError("global certification must run through pdd-sync-checker") + required = { + "wheel_sha256": os.environ.get("PDD_RELEASED_CHECKER_WHEEL_SHA256", ""), + "release_ref": os.environ.get("PDD_RELEASED_CHECKER_REF", ""), + "workflow_identity": os.environ.get( + "PDD_RELEASED_CHECKER_WORKFLOW_IDENTITY", "" + ), + } + if not all(required.values()): + raise ValueError("protected released checker provenance is required") + return CheckerIdentity(**required) diff --git a/pdd/sync_core/certificate_verifier.py b/pdd/sync_core/certificate_verifier.py new file mode 100644 index 000000000..cd8e8f371 --- /dev/null +++ b/pdd/sync_core/certificate_verifier.py @@ -0,0 +1,97 @@ +"""Independent command for accepting a signed exact-SHA global certificate.""" + +from __future__ import annotations + +import argparse +import base64 +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .certificate import CheckerIdentity, verify_global_certificate + + +@dataclass(frozen=True) +class CertificateExpectations: + """Protected acceptance inputs independent of the certificate body.""" + + issuer: str + public_key: bytes + checker: CheckerIdentity + repository_shas: dict[str, tuple[str, str]] + repository_ids: dict[str, str] + + +def _sha(value: Any) -> str: + text = str(value) + if len(text) != 40 or any(character not in "0123456789abcdef" for character in text): + raise ValueError("expected repository SHA is malformed") + return text + + +def load_expectations(path: Path) -> CertificateExpectations: + """Load and strictly validate one protected expectations document.""" + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise ValueError("expectations schema is invalid") + checker_payload = payload["checker"] + repositories = payload["repositories"] + if not isinstance(checker_payload, dict) or not isinstance(repositories, dict): + raise ValueError("expectations payload is malformed") + if set(repositories) != {"pdd", "pdd_cloud"}: + raise ValueError("expectations must name exactly pdd and pdd_cloud") + checker = CheckerIdentity( + str(checker_payload["wheel_sha256"]), + str(checker_payload["release_ref"]), + str(checker_payload["workflow_identity"]), + ) + public_key = base64.b64decode(str(payload["public_key_base64"]), validate=True) + if len(public_key) != 32: + raise ValueError("certificate public key is malformed") + issuer = str(payload["issuer"]) + if not issuer: + raise ValueError("certificate issuer is absent") + repository_shas: dict[str, tuple[str, str]] = {} + repository_ids: dict[str, str] = {} + for name, row in repositories.items(): + if not isinstance(row, dict) or not str(row.get("repository_id", "")): + raise ValueError("expected repository identity is malformed") + repository_shas[name] = (_sha(row["base_sha"]), _sha(row["head_sha"])) + repository_ids[name] = str(row["repository_id"]) + except (KeyError, TypeError, json.JSONDecodeError, OSError) as exc: + raise ValueError("protected certificate expectations are malformed") from exc + return CertificateExpectations( + issuer, public_key, checker, repository_shas, repository_ids + ) + + +def main() -> None: + """Verify one certificate against protected independent expectations.""" + parser = argparse.ArgumentParser() + parser.add_argument("--certificate", type=Path, required=True) + parser.add_argument("--expectations", type=Path, required=True) + arguments = parser.parse_args() + try: + certificate = json.loads(arguments.certificate.read_text(encoding="utf-8")) + if not isinstance(certificate, dict): + raise ValueError("certificate root is not an object") + expected = load_expectations(arguments.expectations) + verified = verify_global_certificate( + certificate, + expected.public_key, + expected_issuer=expected.issuer, + expected_repository_shas=expected.repository_shas, + expected_repository_ids=expected.repository_ids, + expected_checker_identity=expected.checker, + ) + except (OSError, ValueError, json.JSONDecodeError): + verified = False + print(json.dumps({"verified": verified}, sort_keys=True)) + if not verified: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/pdd/sync_core/classifier.py b/pdd/sync_core/classifier.py new file mode 100644 index 000000000..e577c4495 --- /dev/null +++ b/pdd/sync_core/classifier.py @@ -0,0 +1,173 @@ +"""Pure synchronization classifier.""" + +from __future__ import annotations + +from typing import Optional + +from .types import ( + BaselineStatus, + EvidenceOutcome, + FingerprintRecord, + InventoryStatus, + SemanticStatus, + SyncVerdict, + UnitSnapshot, + VerdictDetails, + VerificationProfile, +) +from .trust import ValidationEvidence + + +PROMPT_SIDE_ROLES = frozenset({"prompt", "include", "config", "architecture"}) + + +def _evidence_is_complete( + current: UnitSnapshot, + profile: Optional[VerificationProfile], + evidence: Optional[ValidationEvidence], +) -> bool: + if profile is None or evidence is None: + return False + outcomes = evidence.result_map() + required = (item for item in profile.obligations if item.required) + obligations_pass = all( + outcomes.get(item.obligation_id) is EvidenceOutcome.PASS for item in required + ) + return ( + profile.complete + and profile.unit_id == current.unit_id + and evidence.subject == current.unit_id + and profile.profile_digest == current.verification_profile_digest + and evidence.profile_digest == profile.profile_digest + and evidence.snapshot_digest == current.digest() + and not current.nondeterministic_inputs + and obligations_pass + ) + + +def _preflight_verdict( + current: UnitSnapshot, + baseline: Optional[FingerprintRecord], + inventory: InventoryStatus, + missing: tuple[str, ...], +) -> Optional[SyncVerdict]: + """Return a fail-closed verdict for invalid identity or absent baseline.""" + if inventory is InventoryStatus.INVALID: + statuses = (BaselineStatus.CORRUPT, SemanticStatus.FAILED) + reason = "unit inventory or path policy is invalid" + elif baseline is None: + statuses = (BaselineStatus.UNBASELINED, SemanticStatus.UNKNOWN) + reason = "no trustworthy baseline exists" + elif not baseline.valid: + statuses = (BaselineStatus.CORRUPT, SemanticStatus.FAILED) + reason = "baseline schema or provenance is invalid" + elif baseline.snapshot.unit_id != current.unit_id: + statuses = (BaselineStatus.CORRUPT, SemanticStatus.FAILED) + reason = "baseline identity does not match the current unit" + else: + return None + return SyncVerdict( + current.unit_id, + inventory, + *statuses, + VerdictDetails((), reason, missing), + ) + + +def _changed_roles(current: UnitSnapshot, baseline: UnitSnapshot) -> tuple[str, ...]: + """Compare artifact and closure state without consulting Git provenance.""" + baseline_artifacts = baseline.artifact_map() + current_artifacts = current.artifact_map() + roles = set(baseline_artifacts) | set(current_artifacts) + changed = [ + role + for role in roles + if baseline_artifacts.get(role) != current_artifacts.get(role) + ] + closure_changed = ( + baseline.manifest_digest != current.manifest_digest + or baseline.verification_profile_digest != current.verification_profile_digest + ) + if closure_changed: + closure_roles = ("manifest",) + else: + closure_roles = () + artifact_roles = tuple(role for role, _path in changed) + return tuple(sorted(artifact_roles + closure_roles)) + + +def _drift_verdict( + current: UnitSnapshot, + inventory: InventoryStatus, + changed: tuple[str, ...], +) -> SyncVerdict: + """Classify one-sided drift versus a simultaneous source/derived edit.""" + prompt_changed = bool(set(changed) & PROMPT_SIDE_ROLES) + derived_changed = bool(set(changed) - PROMPT_SIDE_ROLES - {"manifest"}) + conflict = prompt_changed and derived_changed + semantic = SemanticStatus.CONFLICT if conflict else SemanticStatus.UNKNOWN + reason = ( + "prompt-side and derived artifacts changed from the same baseline" + if conflict + else "current state differs from the trusted baseline" + ) + return SyncVerdict( + current.unit_id, + inventory, + BaselineStatus.DRIFTED, + semantic, + VerdictDetails(changed, reason), + ) + + +def classify( + current: UnitSnapshot, + baseline: Optional[FingerprintRecord], + profile: Optional[VerificationProfile], + evidence: Optional[ValidationEvidence], + *, + inventory: InventoryStatus = InventoryStatus.MANAGED, +) -> SyncVerdict: + """Classify a unit without I/O, mutation, subprocesses, or policy inference.""" + missing = tuple( + sorted( + artifact.role + for artifact in current.artifacts + if artifact.required and not artifact.exists + ) + ) + preflight = _preflight_verdict(current, baseline, inventory, missing) + if preflight is not None: + return preflight + assert baseline is not None + changed = _changed_roles(current, baseline.snapshot) + + if missing: + return SyncVerdict( + current.unit_id, + inventory, + BaselineStatus.DRIFTED, + SemanticStatus.FAILED, + VerdictDetails( + changed, + "one or more required artifacts are missing", + missing, + ), + ) + + evidence_complete = _evidence_is_complete(current, profile, evidence) + if not changed: + return SyncVerdict( + current.unit_id, + inventory, + BaselineStatus.CURRENT, + SemanticStatus.VERIFIED if evidence_complete else SemanticStatus.UNKNOWN, + VerdictDetails( + (), + "baseline is current and trusted evidence is complete" + if evidence_complete + else "baseline is current but complete trusted evidence is absent", + evidence_complete=evidence_complete, + ), + ) + return _drift_verdict(current, inventory, changed) diff --git a/pdd/sync_core/durability.py b/pdd/sync_core/durability.py new file mode 100644 index 000000000..280693c2e --- /dev/null +++ b/pdd/sync_core/durability.py @@ -0,0 +1,13 @@ +"""Small filesystem durability primitives shared by canonical state writers.""" + +import os +from pathlib import Path + + +def fsync_directory(path: Path) -> None: + """Flush a directory entry update after atomic replacement.""" + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/pdd/sync_core/evidence_store.py b/pdd/sync_core/evidence_store.py new file mode 100644 index 000000000..d8e68007f --- /dev/null +++ b/pdd/sync_core/evidence_store.py @@ -0,0 +1,204 @@ +"""Serialization and protected-base policy loading for signed evidence.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + +from .trust import ( + AttestationBinding, + AttestationEnvelope, + AttestationError, + AttestationTrustPolicy, + AttestationValidity, + FileReplayStore, +) +from .git_io import read_git_blob +from .types import EvidenceOutcome, ObligationEvidence, UnitId + + +TRUST_POLICY_PATH = PurePosixPath(".pdd/attestation-trust.json") +EVIDENCE_ROOT = PurePosixPath(".pdd/evidence/v2") + + +class EvidenceStoreError(ValueError): + """Raised when evidence or protected trust configuration is malformed.""" + + +@dataclass(frozen=True) +class LoadedTrustPolicy: + """Runtime verifier and deterministic protected policy digest.""" + + verifier: AttestationTrustPolicy + digest: str + issuer_ids: tuple[str, ...] + + +def evidence_relpath(attestation_id: str) -> PurePosixPath: + """Return the safe repository-relative location for one attestation.""" + safe = "-_.0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + if not attestation_id or any(character not in safe for character in attestation_id): + raise EvidenceStoreError("attestation ID contains unsafe characters") + return EVIDENCE_ROOT / f"{attestation_id}.json" + + +def attestation_payload(envelope: AttestationEnvelope) -> dict[str, Any]: + """Convert a signed envelope to stable JSON data without altering its payload.""" + binding = envelope.binding + return { + "attestation_id": envelope.attestation_id, + "issuer": envelope.issuer, + "binding": { + "subject": { + "repository_id": binding.subject.repository_id, + "prompt_relpath": binding.subject.prompt_relpath.as_posix(), + "language_id": binding.subject.language_id, + }, + "snapshot_digest": binding.snapshot_digest, + "profile_digest": binding.profile_digest, + "runner_digest": binding.runner_digest, + "tool_version": binding.tool_version, + "base_sha": binding.base_sha, + "checked_sha": binding.checked_sha, + }, + "results": [ + { + "obligation_id": result.obligation_id, + "outcome": result.outcome.value, + } + for result in envelope.results + ], + "validity": { + "issued_at": envelope.validity.issued_at, + "expires_at": envelope.validity.expires_at, + "nonce": envelope.validity.nonce, + }, + "signature": envelope.signature, + } + + +def encode_attestation(envelope: AttestationEnvelope) -> bytes: + """Encode a signed envelope for transactional repository persistence.""" + return json.dumps( + attestation_payload(envelope), sort_keys=True, indent=2 + ).encode("utf-8") + b"\n" + + +def _string(payload: Mapping[str, Any], key: str) -> str: + value = payload.get(key) + if not isinstance(value, str) or not value: + raise EvidenceStoreError(f"evidence field {key!r} must be a non-empty string") + return value + + +def decode_attestation(payload: Mapping[str, Any]) -> AttestationEnvelope: + """Decode untrusted JSON into a typed envelope without granting trust.""" + try: + binding_data = payload["binding"] + subject_data = binding_data["subject"] + validity_data = payload["validity"] + results_data = payload["results"] + if not all(isinstance(item, dict) for item in results_data): + raise TypeError("result must be an object") + subject = UnitId( + _string(subject_data, "repository_id"), + PurePosixPath(_string(subject_data, "prompt_relpath")), + _string(subject_data, "language_id"), + ) + binding = AttestationBinding( + subject, + _string(binding_data, "snapshot_digest"), + _string(binding_data, "profile_digest"), + _string(binding_data, "runner_digest"), + _string(binding_data, "tool_version"), + _string(binding_data, "base_sha"), + _string(binding_data, "checked_sha"), + ) + results = tuple( + ObligationEvidence( + _string(item, "obligation_id"), + EvidenceOutcome(_string(item, "outcome")), + ) + for item in results_data + ) + validity = AttestationValidity( + _string(validity_data, "issued_at"), + _string(validity_data, "expires_at"), + _string(validity_data, "nonce"), + ) + return AttestationEnvelope( + _string(payload, "attestation_id"), + _string(payload, "issuer"), + binding, + results, + validity, + _string(payload, "signature"), + ) + except (KeyError, TypeError, ValueError) as exc: + raise EvidenceStoreError(f"attestation envelope is malformed: {exc}") from exc + + +def load_attestation(root: Path, attestation_id: str) -> AttestationEnvelope: + """Load an untrusted candidate envelope from its canonical path.""" + path = Path(root).resolve().joinpath(*evidence_relpath(attestation_id).parts) + if path.is_symlink() or not path.is_file(): + raise EvidenceStoreError(f"attestation file is missing or unsafe: {path}") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise EvidenceStoreError(f"attestation JSON is malformed: {path}") from exc + if not isinstance(payload, dict): + raise EvidenceStoreError("attestation root must be an object") + envelope = decode_attestation(payload) + if envelope.attestation_id != attestation_id: + raise EvidenceStoreError("attestation path and embedded identity differ") + return envelope + + +def load_trust_policy( + root: Path, + protected_base_ref: str, + *, + replay_ledger_path: Path, +) -> LoadedTrustPolicy: + """Load issuer and revocation authority only from the protected base tree.""" + raw = read_git_blob(Path(root), protected_base_ref, TRUST_POLICY_PATH) + if raw is None: + raise EvidenceStoreError("protected base has no attestation trust policy") + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise EvidenceStoreError("protected attestation trust policy is malformed") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("issuers"), list): + raise EvidenceStoreError("protected trust policy issuers must be a list") + issuers: dict[str, bytes] = {} + for item in payload["issuers"]: + if not isinstance(item, dict): + raise EvidenceStoreError("protected issuer entry must be an object") + issuer_id = _string(item, "issuer_id") + try: + public_key = base64.b64decode(_string(item, "public_key"), validate=True) + except ValueError as exc: + raise EvidenceStoreError("protected issuer public key is malformed") from exc + if len(public_key) != 32 or issuer_id in issuers: + raise EvidenceStoreError("protected issuer is duplicate or not Ed25519") + issuers[issuer_id] = public_key + revoked_issuers = frozenset(payload.get("revoked_issuers", [])) + revoked_attestations = frozenset(payload.get("revoked_attestations", [])) + if not all(isinstance(item, str) for item in revoked_issuers | revoked_attestations): + raise EvidenceStoreError("protected revocation entries must be strings") + digest = hashlib.sha256(raw).hexdigest() + try: + verifier = AttestationTrustPolicy( + issuers, + revoked_issuers=revoked_issuers, + revoked_attestations=revoked_attestations, + replay_store=FileReplayStore(replay_ledger_path), + ) + except AttestationError as exc: + raise EvidenceStoreError("protected trust policy cannot be initialized") from exc + return LoadedTrustPolicy(verifier, digest, tuple(sorted(issuers))) diff --git a/pdd/sync_core/finalize.py b/pdd/sync_core/finalize.py new file mode 100644 index 000000000..49597f8ed --- /dev/null +++ b/pdd/sync_core/finalize.py @@ -0,0 +1,314 @@ +"""Trusted validation and transactional canonical fingerprint finalization.""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import subprocess +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + +from .evidence_store import ( + encode_attestation, + evidence_relpath, + load_attestation, + load_trust_policy, +) +from .fingerprint_store import FingerprintStore, encode_fingerprint +from .manifest import build_unit_manifest +from .runner import ( + TRUSTED_RUNNER_VERSION, + AttestationIssue, + RunBinding, + RunnerConfig, + run_profile, + runner_identity_digest, +) +from .snapshot import build_unit_snapshot +from .transaction import ( + FileState, + PlannedWrite, + TransactionManager, + TransactionPhase, + TransactionResult, +) +from .trust import AttestationBinding, AttestationSigner +from .types import ( + EvidenceOutcome, + FingerprintProvenance, + FingerprintRecord, + SemanticStatus, +) +from .verification import load_verification_profiles + + +@dataclass(frozen=True) +class FinalizeResult: + """Committed trusted finalization and its signed attestation identity.""" + + transaction: TransactionResult + attestation_id: str + fingerprint_path: PurePosixPath + + +class CanonicalFinalizationError(RuntimeError): + """Raised when an opted-in mutation cannot commit trusted final state.""" + + +def canonical_root_for_paths(paths: dict[str, Path] | None) -> Path | None: + """Return the opted-in Git root for legacy path-based callers.""" + # pylint: disable=import-outside-toplevel + from ..continuous_sync import canonical_sync_enabled, repository_root + + start = Path(paths.get("prompt", Path.cwd())) if paths else Path.cwd() + root = repository_root(start) + return root if canonical_sync_enabled(root) else None + + +def finalize_legacy_paths(paths: dict[str, Path] | None) -> bool: + """Route a legacy fingerprint request through trusted canonical finalization.""" + root = canonical_root_for_paths(paths) + if root is None: + return False + try: + if not paths or "prompt" not in paths: + raise ValueError("canonical finalization requires an exact prompt path") + protected_base = os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") + if not protected_base: + raise ValueError("canonical sync requires PDD_SYNC_PROTECTED_BASE_SHA") + prompt = Path(paths["prompt"]).resolve() + module = PurePosixPath(prompt.relative_to(root).as_posix()) + finalize_unit( + root, + module, + base_ref=protected_base, + head_ref="HEAD", + signer=attestation_signer_from_environment(), + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CanonicalFinalizationError( + f"trusted canonical finalization failed: {exc}" + ) from exc + return True + + +def _git_sha(root: Path, ref: str) -> str: + result = subprocess.run( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise ValueError(f"cannot resolve Git commit: {ref}") + return result.stdout.strip() + + +def attestation_signer_from_environment() -> AttestationSigner: + """Load the protected semantic-evidence signing identity.""" + encoded = os.environ.get("PDD_ATTESTATION_SIGNING_KEY") + issuer = os.environ.get("PDD_ATTESTATION_ISSUER", "trusted-sync-runner") + if not encoded: + raise ValueError("PDD_ATTESTATION_SIGNING_KEY is required") + try: + key = base64.b64decode(encoded, validate=True) + except ValueError as exc: + raise ValueError("PDD_ATTESTATION_SIGNING_KEY is malformed") from exc + return AttestationSigner(issuer, key) + + +def _artifact_writes(root: Path, snapshot) -> tuple[PlannedWrite, ...]: + writes = [] + for artifact in snapshot.artifacts: + if not artifact.exists or artifact.git_mode not in {"100644", "100755"}: + raise ValueError( + f"cannot transactionally finalize artifact: {artifact.relpath}" + ) + path = root.joinpath(*artifact.relpath.parts) + content = path.read_bytes() + if hashlib.sha256(content).hexdigest() != artifact.digest: + raise ValueError(f"artifact changed while snapshotting: {artifact.relpath}") + writes.append( + PlannedWrite( + artifact.relpath, + content, + artifact.git_mode, + expected=FileState( + True, artifact.digest, artifact.git_mode, "regular" + ), + ) + ) + return tuple(writes) + + +def _external_replay_ledger(root: Path, configured: Path | None) -> Path: + value = configured or ( + Path(os.environ["PDD_ATTESTATION_REPLAY_LEDGER"]) + if os.environ.get("PDD_ATTESTATION_REPLAY_LEDGER") + else None + ) + if value is None: + raise ValueError("canonical finalization requires an external replay ledger") + replay = value.expanduser().resolve() + try: + replay.relative_to(root) + except ValueError: + return replay + raise ValueError("attestation replay ledger must be outside the candidate checkout") + + +def _reusable_result( + root: Path, + snapshot, + profile, + store: FingerprintStore, + verifier, + *, + base_sha: str, + head_sha: str, + now: datetime, +) -> FinalizeResult | None: + # pylint: disable=too-many-arguments,too-many-locals + baseline = store.load(snapshot.unit_id) + if ( + baseline is None + or baseline.snapshot != snapshot + or baseline.claimed_semantic_status is not SemanticStatus.VERIFIED + or not baseline.attestation_ref + ): + return None + envelope = load_attestation(root, baseline.attestation_ref) + binding = AttestationBinding( + snapshot.unit_id, + snapshot.digest(), + profile.profile_digest, + runner_identity_digest(profile, root=root, ref=head_sha), + TRUSTED_RUNNER_VERSION, + base_sha, + envelope.binding.checked_sha, + ) + verifier.verify_current_for_idempotency(envelope, binding, now=now) + ancestry = subprocess.run( + ["git", "merge-base", "--is-ancestor", binding.checked_sha, head_sha], + cwd=root, + capture_output=True, + check=False, + ) + required = {item.obligation_id for item in profile.obligations if item.required} + passed = { + item.obligation_id + for item in envelope.results + if item.outcome is EvidenceOutcome.PASS + } + if ancestry.returncode != 0 or required != passed: + return None + fingerprint = PurePosixPath(store.path_for(snapshot.unit_id).relative_to(root).as_posix()) + transaction = TransactionResult( + baseline.provenance.transaction_id, + TransactionPhase.COMMITTED, + (), + True, + ) + return FinalizeResult(transaction, envelope.attestation_id, fingerprint) + + +def finalize_unit( + root: Path, + module: PurePosixPath, + *, + base_ref: str, + head_ref: str, + signer: AttestationSigner, + config: RunnerConfig = RunnerConfig(), + replay_ledger_path: Path | None = None, +) -> FinalizeResult: + # pylint: disable=too-many-arguments,too-many-locals + """Validate one complete unit and atomically finalize all trusted state.""" + repository_root = Path(root).resolve() + base_sha = _git_sha(repository_root, base_ref) + head_sha = _git_sha(repository_root, head_ref) + if _git_sha(repository_root, "HEAD") != head_sha: + raise ValueError("canonical finalization requires HEAD at the checked SHA") + manifest = build_unit_manifest( + repository_root, base_ref=base_sha, head_ref=head_sha + ) + matches = [ + unit + for unit in manifest.managed_units + if unit.unit_id.prompt_relpath == module + ] + if len(matches) != 1: + raise ValueError(f"finalization requires exactly one managed unit: {module}") + profiles = load_verification_profiles(repository_root, manifest) + profile = profiles.for_unit(matches[0].unit_id) + if profile is None or not profile.complete: + raise ValueError("finalization requires a complete protected profile") + snapshot = build_unit_snapshot(repository_root, manifest, matches[0], profile) + artifact_writes = _artifact_writes(repository_root, snapshot) + now = datetime.now(timezone.utc) + replay = _external_replay_ledger(repository_root, replay_ledger_path) + verifier = load_trust_policy( + repository_root, base_sha, replay_ledger_path=replay + ).verifier + reusable = _reusable_result( + repository_root, + snapshot, + profile, + FingerprintStore(repository_root), + verifier, + base_sha=base_sha, + head_sha=head_sha, + now=now, + ) + if reusable is not None: + return reusable + transaction_id = f"finalize-{uuid.uuid4()}" + attestation_id = f"attestation-{uuid.uuid4()}" + envelope, executions = run_profile( + repository_root, + profile, + RunBinding(snapshot.digest(), base_sha, head_sha), + AttestationIssue(signer, attestation_id, str(uuid.uuid4()), now), + config, + ) + failures = [item for item in executions if item.outcome is not EvidenceOutcome.PASS] + if failures: + detail = "; ".join( + f"{item.obligation_id}={item.outcome.value}" for item in failures + ) + raise ValueError(f"trusted validation did not pass: {detail}") + post_run_snapshot = build_unit_snapshot( + repository_root, manifest, matches[0], profile + ) + if post_run_snapshot != snapshot: + raise ValueError("artifact closure changed during trusted validation") + verifier.verify(envelope, envelope.binding, now=now) + provenance = FingerprintProvenance( + "trusted-validation", + f"pdd validate --module {module.as_posix()}", + transaction_id, + head_sha, + now.isoformat(), + TRUSTED_RUNNER_VERSION, + ) + record = FingerprintRecord( + snapshot, 2, 2, provenance, SemanticStatus.VERIFIED, attestation_id + ) + store = FingerprintStore(repository_root) + store.validate(record) + fingerprint = PurePosixPath( + store.path_for(record.snapshot.unit_id).relative_to(repository_root).as_posix() + ) + writes = ( + *artifact_writes, + PlannedWrite(evidence_relpath(attestation_id), encode_attestation(envelope), "100644"), + PlannedWrite(fingerprint, encode_fingerprint(record), "100644"), + ) + manager = TransactionManager(repository_root) + manager.prepare(transaction_id, writes) + transaction = manager.commit(transaction_id) + return FinalizeResult(transaction, attestation_id, fingerprint) diff --git a/pdd/sync_core/fingerprint_store.py b/pdd/sync_core/fingerprint_store.py new file mode 100644 index 000000000..4765f3a63 --- /dev/null +++ b/pdd/sync_core/fingerprint_store.py @@ -0,0 +1,291 @@ +"""Versioned canonical fingerprint persistence and legacy read support.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Mapping, Optional + +from filelock import FileLock + +from .durability import fsync_directory +from .identity import read_repository_identity +from .types import ( + ArtifactSnapshot, + FingerprintProvenance, + FingerprintRecord, + SemanticStatus, + UnitId, + UnitSnapshot, +) + + +class FingerprintStoreError(ValueError): + """Raised when canonical fingerprint state cannot be validated or persisted.""" + + +class CorruptFingerprintError(FingerprintStoreError): + """Raised when stored state exists but is malformed or inconsistent.""" + + +@dataclass(frozen=True) +class LegacyFingerprintRecord: + """Readable legacy metadata that cannot certify canonical synchronization.""" + + path: Path + payload: Mapping[str, Any] + + +def _unit_payload(unit_id: UnitId) -> dict[str, str]: + return { + "repository_id": unit_id.repository_id, + "prompt_relpath": unit_id.prompt_relpath.as_posix(), + "language_id": unit_id.language_id, + } + + +def _record_payload(record: FingerprintRecord) -> dict[str, Any]: + return { + "schema_version": record.schema_version, + "hash_algorithm_version": record.hash_algorithm_version, + "unit_id": _unit_payload(record.snapshot.unit_id), + "artifacts": [ + { + "role": item.role, + "path": item.relpath.as_posix(), + "hash": item.digest, + "git_mode": item.git_mode, + "required": item.required, + } + for item in sorted(record.snapshot.artifacts) + ], + "manifest_digest": record.snapshot.manifest_digest, + "dependency_snapshot_digest": record.snapshot.dependency_snapshot_digest, + "verification_profile_digest": record.snapshot.verification_profile_digest, + "nondeterministic_inputs": record.snapshot.nondeterministic_inputs, + "provenance": { + "kind": record.provenance.kind, + "command": record.provenance.command, + "transaction_id": record.provenance.transaction_id, + "git_commit": record.provenance.git_commit, + "timestamp": record.provenance.timestamp, + "pdd_version": record.provenance.pdd_version, + "reviewed_by": record.provenance.reviewed_by, + "reason": record.provenance.reason, + }, + "claimed_semantic_status": record.claimed_semantic_status.value, + "attestation_ref": record.attestation_ref, + } + + +def encode_fingerprint(record: FingerprintRecord) -> bytes: + """Encode a validated record for inclusion in a larger transaction.""" + return json.dumps( + _record_payload(record), sort_keys=True, indent=2 + ).encode("utf-8") + b"\n" + + +def _required(payload: Mapping[str, Any], key: str, expected_type: type) -> Any: + value = payload.get(key) + if not isinstance(value, expected_type): + raise CorruptFingerprintError(f"fingerprint field {key!r} has invalid type") + return value + + +def _parse_record(payload: Mapping[str, Any]) -> FingerprintRecord: + try: + unit_payload = _required(payload, "unit_id", dict) + unit_id = UnitId( + _required(unit_payload, "repository_id", str), + PurePosixPath(_required(unit_payload, "prompt_relpath", str)), + _required(unit_payload, "language_id", str), + ) + artifacts_payload = _required(payload, "artifacts", list) + artifacts = tuple( + ArtifactSnapshot( + _required(item, "role", str), + PurePosixPath(_required(item, "path", str)), + item.get("hash"), + item.get("git_mode"), + item.get("required", True), + ) + for item in artifacts_payload + if isinstance(item, dict) + ) + if len(artifacts) != len(artifacts_payload): + raise CorruptFingerprintError("fingerprint artifact entry is not an object") + snapshot = UnitSnapshot( + unit_id, + artifacts, + _required(payload, "manifest_digest", str), + _required(payload, "dependency_snapshot_digest", str), + _required(payload, "verification_profile_digest", str), + bool(payload.get("nondeterministic_inputs", False)), + ) + provenance_payload = _required(payload, "provenance", dict) + reviewed_by = provenance_payload.get("reviewed_by") + reason = provenance_payload.get("reason") + if reviewed_by is not None and not isinstance(reviewed_by, str): + raise CorruptFingerprintError("fingerprint reviewer must be a string or null") + if reason is not None and not isinstance(reason, str): + raise CorruptFingerprintError("fingerprint reason must be a string or null") + provenance = FingerprintProvenance( + _required(provenance_payload, "kind", str), + _required(provenance_payload, "command", str), + _required(provenance_payload, "transaction_id", str), + _required(provenance_payload, "git_commit", str), + _required(provenance_payload, "timestamp", str), + _required(provenance_payload, "pdd_version", str), + reviewed_by, + reason, + ) + claimed = SemanticStatus(_required(payload, "claimed_semantic_status", str)) + attestation_ref = payload.get("attestation_ref") + if attestation_ref is not None and not isinstance(attestation_ref, str): + raise CorruptFingerprintError("attestation_ref must be a string or null") + return FingerprintRecord( + snapshot, + _required(payload, "schema_version", int), + _required(payload, "hash_algorithm_version", int), + provenance, + claimed, + attestation_ref, + ) + except (TypeError, ValueError) as exc: + if isinstance(exc, CorruptFingerprintError): + raise + raise CorruptFingerprintError(f"fingerprint payload is invalid: {exc}") from exc + + +class FingerprintStore: + """Locked atomic store for canonical v2 fingerprint records.""" + + def __init__(self, checkout_root: Path) -> None: + self.checkout_root = Path(checkout_root).resolve() + self.repository_id = read_repository_identity( + self.checkout_root, require_persistent=True + ).value + self.meta_dir = self.checkout_root / ".pdd/meta/v2" + self.lock_dir = self.checkout_root / ".pdd/locks/fingerprints" + + @staticmethod + def _key(unit_id: UnitId) -> str: + payload = json.dumps(_unit_payload(unit_id), sort_keys=True).encode() + return hashlib.sha256(payload).hexdigest() + + def path_for(self, unit_id: UnitId) -> Path: + """Return the collision-resistant canonical path for one unit identity.""" + return self.meta_dir / f"{self._key(unit_id)}.json" + + def _ensure_state_directory(self, directory: Path, mode: int) -> None: + current = self.checkout_root + for part in directory.relative_to(self.checkout_root).parts: + current = current / part + if current.exists() or current.is_symlink(): + current_mode = current.lstat().st_mode + if stat.S_ISLNK(current_mode) or not stat.S_ISDIR(current_mode): + raise FingerprintStoreError(f"state directory is unsafe: {current}") + else: + current.mkdir(mode=mode) + + def validate(self, record: FingerprintRecord) -> None: + """Validate a record before direct or transactional persistence.""" + if record.snapshot.unit_id.repository_id != self.repository_id: + raise FingerprintStoreError("fingerprint repository identity does not match") + if not record.valid: + raise FingerprintStoreError("only complete canonical v2 records may be written") + missing = [ + item.role + for item in record.snapshot.artifacts + if item.required and not item.exists + ] + if missing: + raise FingerprintStoreError( + "required artifacts have null hash or mode: " + ", ".join(sorted(missing)) + ) + if ( + record.claimed_semantic_status is SemanticStatus.VERIFIED + and not record.attestation_ref + ): + raise FingerprintStoreError("VERIFIED fingerprint requires an attestation") + if ( + record.provenance.kind == "baseline-reset" + and record.claimed_semantic_status is not SemanticStatus.UNKNOWN + ): + raise FingerprintStoreError("baseline reset must remain semantic UNKNOWN") + if record.provenance.kind == "baseline-reset" and ( + not record.provenance.reviewed_by or not record.provenance.reason + ): + raise FingerprintStoreError( + "baseline reset requires a recorded reviewer and rationale" + ) + + def load(self, unit_id: UnitId) -> Optional[FingerprintRecord]: + """Load and validate one canonical record without mutating state.""" + path = self.path_for(unit_id) + if not path.exists(): + return None + if path.is_symlink() or not path.is_file(): + raise CorruptFingerprintError("fingerprint path is not a regular file") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise CorruptFingerprintError(f"cannot decode fingerprint: {path}") from exc + if not isinstance(payload, dict): + raise CorruptFingerprintError("fingerprint root must be an object") + record = _parse_record(payload) + if record.snapshot.unit_id != unit_id: + raise CorruptFingerprintError("fingerprint key and embedded identity differ") + self.validate(record) + return record + + def write(self, record: FingerprintRecord) -> Path: + """Atomically replace one validated record while holding its process lock.""" + self.validate(record) + self._ensure_state_directory(self.meta_dir, 0o755) + self._ensure_state_directory(self.lock_dir, 0o700) + path = self.path_for(record.snapshot.unit_id) + lock_path = self.lock_dir / f"{path.stem}.lock" + encoded = encode_fingerprint(record) + with FileLock(str(lock_path)): + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=self.meta_dir + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o644) + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + fsync_directory(self.meta_dir) + except BaseException: + temporary.unlink(missing_ok=True) + raise + return path + + def read_legacy(self, path: Path) -> LegacyFingerprintRecord: + """Read legacy JSON for migration without granting it v2 authority.""" + candidate = Path(path) + if not candidate.is_absolute(): + candidate = self.checkout_root / candidate + resolved = candidate.resolve(strict=True) + try: + resolved.relative_to(self.checkout_root) + except ValueError as exc: + raise FingerprintStoreError("legacy fingerprint escapes checkout") from exc + if candidate.is_symlink() or not resolved.is_file(): + raise FingerprintStoreError("legacy fingerprint is not a regular file") + try: + payload = json.loads(resolved.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise CorruptFingerprintError("legacy fingerprint is malformed") from exc + if not isinstance(payload, dict): + raise CorruptFingerprintError("legacy fingerprint root must be an object") + return LegacyFingerprintRecord(resolved, payload) diff --git a/pdd/sync_core/git_io.py b/pdd/sync_core/git_io.py new file mode 100644 index 000000000..6adde8424 --- /dev/null +++ b/pdd/sync_core/git_io.py @@ -0,0 +1,15 @@ +"""Read-only Git object helpers for protected base/head evaluation.""" + +import subprocess +from pathlib import Path, PurePosixPath + + +def read_git_blob(root: Path, ref: str, path: PurePosixPath) -> bytes | None: + """Read a blob from an immutable tree, returning None when it is absent.""" + result = subprocess.run( + ["git", "show", f"{ref}:{path.as_posix()}"], + cwd=root, + capture_output=True, + check=False, + ) + return result.stdout if result.returncode == 0 else None diff --git a/pdd/sync_core/identity.py b/pdd/sync_core/identity.py new file mode 100644 index 000000000..c4295b05c --- /dev/null +++ b/pdd/sync_core/identity.py @@ -0,0 +1,97 @@ +"""Stable repository identity used by canonical sync records.""" + +from __future__ import annotations + +import os +import subprocess +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from .durability import fsync_directory + + +REPOSITORY_ID_RELPATH = Path(".pdd/repository-id") + + +class RepositoryIdentityError(ValueError): + """Raised when protected repository identity is absent or malformed.""" + + +@dataclass(frozen=True) +class RepositoryIdentity: + """A repository UUID and whether it is safe for persistent state.""" + + value: str + persistent: bool + + +def canonical_repository_id(value: str) -> str: + """Validate and normalize one protected repository UUID.""" + try: + parsed = uuid.UUID(value.strip()) + except (AttributeError, ValueError) as exc: + raise RepositoryIdentityError("repository ID must be a valid UUID") from exc + if str(parsed) != value.strip().lower(): + raise RepositoryIdentityError("repository ID must use canonical lowercase UUID form") + return str(parsed) + + +def _ephemeral_seed(root: Path) -> str: + result = subprocess.run( + ["git", "config", "--get", "remote.origin.url"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() or str(root.resolve()) + + +def read_repository_identity(root: Path, *, require_persistent: bool = False) -> RepositoryIdentity: + """Read protected identity or return a report-only ephemeral UUID.""" + repository_root = Path(root).resolve() + identity_path = repository_root / REPOSITORY_ID_RELPATH + if identity_path.exists(): + if identity_path.is_symlink() or not identity_path.is_file(): + raise RepositoryIdentityError("repository ID must be a regular file") + return RepositoryIdentity( + canonical_repository_id(identity_path.read_text(encoding="ascii")), + True, + ) + if require_persistent: + raise RepositoryIdentityError( + "canonical state requires initialization of .pdd/repository-id" + ) + ephemeral = uuid.uuid5(uuid.NAMESPACE_URL, f"pdd-legacy:{_ephemeral_seed(repository_root)}") + return RepositoryIdentity(str(ephemeral), False) + + +def initialize_repository_identity( + root: Path, + repository_id: Optional[str] = None, +) -> RepositoryIdentity: + """Create the protected repository UUID exactly once using an atomic write.""" + repository_root = Path(root).resolve() + identity_path = repository_root / REPOSITORY_ID_RELPATH + if identity_path.exists(): + return read_repository_identity(repository_root, require_persistent=True) + value = canonical_repository_id(repository_id or str(uuid.uuid4())) + state_dir = identity_path.parent + if state_dir.exists() and (state_dir.is_symlink() or not state_dir.is_dir()): + raise RepositoryIdentityError(".pdd must be a real directory") + state_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = state_dir / f".{identity_path.name}.{os.getpid()}.tmp" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="ascii") as handle: + handle.write(value + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, identity_path) + fsync_directory(state_dir) + except BaseException: + temporary.unlink(missing_ok=True) + raise + return RepositoryIdentity(value, True) diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py new file mode 100644 index 000000000..efe0ac30c --- /dev/null +++ b/pdd/sync_core/includes.py @@ -0,0 +1,370 @@ +"""Canonical parser for every supported prompt include syntax.""" + +from __future__ import annotations + +import re +import hashlib +import json +import posixpath +from dataclasses import dataclass +from enum import Enum +from pathlib import PurePosixPath + +from .path_policy import PathPolicy, PathPolicyError + + +class IncludeSyntax(str, Enum): + """Source grammar that produced an include reference.""" + + XML = "xml" + XML_MANY = "xml-many" + BACKTICK = "backtick" + + +@dataclass(frozen=True, order=True) +class IncludeReference: + """One ordered include declaration and its behavior-bearing attributes.""" + + position: int + path: str + syntax: IncludeSyntax + select: str | None = None + query: str | None = None + optional: bool = False + expand_dependencies: bool = False + + +class IncludeGraphError(ValueError): + """Raised when an include closure is missing, cyclic, or policy-invalid.""" + + +@dataclass(frozen=True, order=True) +class IncludeEdge: + """Resolved dependency edge including behavior-bearing parser attributes.""" + + source: PurePosixPath + target: PurePosixPath + reference: IncludeReference + target_exists: bool + + +@dataclass(frozen=True, order=True) +class IncludedArtifact: + """Content and mode snapshot for one resolved expansion input.""" + + relpath: PurePosixPath + digest: str + git_mode: str + + +@dataclass(frozen=True) +class IncludeClosure: + """Deterministic transitive expansion closure for one prompt.""" + + root: PurePosixPath + artifacts: tuple[IncludedArtifact, ...] + edges: tuple[IncludeEdge, ...] + has_nondeterministic_query: bool + + def digest(self) -> str: + """Hash all resolved bytes, modes, edges, and include attributes.""" + payload = { + "root": self.root.as_posix(), + "artifacts": [ + { + "path": item.relpath.as_posix(), + "digest": item.digest, + "git_mode": item.git_mode, + } + for item in self.artifacts + ], + "edges": [ + { + "source": edge.source.as_posix(), + "target": edge.target.as_posix(), + "path": edge.reference.path, + "syntax": edge.reference.syntax.value, + "select": edge.reference.select, + "query": edge.reference.query, + "optional": edge.reference.optional, + "expand_dependencies": edge.reference.expand_dependencies, + "target_exists": edge.target_exists, + } + for edge in self.edges + ], + "has_nondeterministic_query": self.has_nondeterministic_query, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +_INCLUDE_PATTERN = re.compile( + r'\s[^>]*?)?(?' + r'\s*(?P[^<>\r\n]+?)\s*' + r'|\s[^>]*?)/>', +) +_INCLUDE_MANY_PATTERN = re.compile( + r'\s[^>]*+)?>(?P.*?)', + re.DOTALL, +) +_BACKTICK_PATTERN = re.compile(r"```<([^>]*+)>```") +_ATTRIBUTE_PATTERN = re.compile(r'(\w+)\s*=\s*["\']([^"\']*)["\']') + + +def _parse_attrs(raw: str) -> dict[str, str]: + attrs = {match.group(1): match.group(2) for match in _ATTRIBUTE_PATTERN.finditer(raw)} + for boolean_name in ("optional", "expand"): + pattern = rf"(? bool: + return value is not None and value.strip().casefold() not in { + "", + "0", + "false", + "no", + "off", + } + + +def parse_include_references(text: str) -> tuple[IncludeReference, ...]: + """Parse includes once, preserving duplicates and deterministic source order.""" + if not text: + return () + references: list[IncludeReference] = [] + for match in _INCLUDE_PATTERN.finditer(text): + attrs = _parse_attrs(match.group("attrs") or match.group("attrs_self") or "") + body = match.group("content") or "" + path = (attrs.get("path") or body).strip() + if path: + references.append( + IncludeReference( + match.start(), + path, + IncludeSyntax.XML, + attrs.get("select"), + attrs.get("query"), + _enabled(attrs.get("optional")), + _enabled(attrs.get("expand")), + ) + ) + for match in _INCLUDE_MANY_PATTERN.finditer(text): + attrs = _parse_attrs(match.group("attrs") or "") + values = ( + item.strip() + for line in match.group("inner").splitlines() + for item in line.split(",") + ) + for offset, path in enumerate(value for value in values if value): + references.append( + IncludeReference( + match.start() + offset, + path, + IncludeSyntax.XML_MANY, + optional=_enabled(attrs.get("optional")), + expand_dependencies=_enabled(attrs.get("expand")), + ) + ) + for match in _BACKTICK_PATTERN.finditer(text): + path = match.group(1).strip() + if path: + references.append( + IncludeReference(match.start(), path, IncludeSyntax.BACKTICK) + ) + return tuple(sorted(references)) + + +def include_paths(text: str) -> set[str]: + """Return path membership for preprocessing's user-intent guard.""" + return {reference.path for reference in parse_include_references(text)} + + +def _normalized(path: PurePosixPath, raw_path: str) -> PurePosixPath: + normalized = PurePosixPath(posixpath.normpath(path.as_posix())) + if normalized.is_absolute() or ".." in normalized.parts: + raise IncludeGraphError(f"include path escapes repository: {raw_path}") + return normalized + + +def _candidate_paths( + source: PurePosixPath, + raw_path: str, + aliases: tuple[PurePosixPath, ...] = (), +) -> tuple[PurePosixPath, ...]: + # pylint: disable=too-many-branches,too-many-locals + declared = PurePosixPath(raw_path) + if declared.is_absolute(): + raise IncludeGraphError(f"absolute include path is not allowed: {raw_path}") + candidates = [source.parent / declared] + candidates.extend(alias.parent / declared for alias in aliases) + leading_parents = 0 + for part in declared.parts: + if part != "..": + break + leading_parents += 1 + if leading_parents: + repository_tail = PurePosixPath(*declared.parts[leading_parents:]) + if repository_tail.parts: + candidates.append(repository_tail) + prompt_index = ( + max(index for index, part in enumerate(source.parts) if part == "prompts") + if "prompts" in source.parts + else None + ) + project_prefix = PurePosixPath(".") + project_namespace = None + if prompt_index is not None: + project_prefix = PurePosixPath(*source.parts[:prompt_index]) + remainder = source.parts[prompt_index + 1 :] + if remainder: + project_namespace = remainder[0] + projected = PurePosixPath( + *source.parts[:prompt_index], *remainder + ) + candidates.append(projected.parent / declared) + if ".." not in declared.parts: + candidates.append(declared) + candidates.append(project_prefix / declared) + candidates.append(project_prefix / "prompts" / declared) + if project_namespace: + candidates.append(project_prefix / project_namespace / declared) + candidates.extend(parent / declared for parent in source.parents) + if raw_path.startswith("@/"): + alias = PurePosixPath(raw_path[2:]) + if project_namespace: + candidates.append( + project_prefix / project_namespace / "src" / alias + ) + candidates.append(project_prefix / "src" / alias) + for parent in source.parents: + candidates.extend((parent / "src" / alias, parent / "frontend/src" / alias)) + normalized: list[PurePosixPath] = [] + for candidate in candidates: + try: + path = _normalized(candidate, raw_path) + except IncludeGraphError: + continue + variants = [path] + if not path.suffix and not any(character in path.name for character in "*?["): + suffixes = (".py", ".ts", ".tsx", ".js", ".jsx") + variants.extend(path.with_suffix(suffix) for suffix in suffixes) + normalized.extend(variants) + unique = tuple(dict.fromkeys(normalized)) + if not unique: + raise IncludeGraphError(f"include path escapes repository: {raw_path}") + return unique + + +def _resolved_targets( + source: PurePosixPath, + reference: IncludeReference, + policy: PathPolicy, + aliases: tuple[PurePosixPath, ...] = (), +) -> tuple[PurePosixPath, ...]: + candidates = _candidate_paths(source, reference.path, aliases) + wildcard = any(character in reference.path for character in "*?[") + for candidate in candidates: + if wildcard: + matches = tuple( + sorted( + PurePosixPath(path.relative_to(policy.checkout_root).as_posix()) + for path in policy.checkout_root.glob(candidate.as_posix()) + if path.is_file() + ) + ) + if matches: + return matches + continue + try: + policy.resolve(candidate) + return (candidate,) + except (FileNotFoundError, PathPolicyError): + continue + return (candidates[0],) + + +def _external_package_reference(raw_path: str) -> bool: + """Return whether an unresolved include names a package, not a repo path.""" + path = PurePosixPath(raw_path.split(" ", 1)[0]) + local_prefixes = { + "@", "app", "backend", "components", "context", "docs", "frontend", "pdd", "src" + } + return ( + not path.suffix + and not raw_path.endswith("/") + and ".." not in path.parts + and path.parts[0] not in local_prefixes + ) + + +def build_include_closure( + root: PurePosixPath, + policy: PathPolicy, + *, + root_aliases: tuple[PurePosixPath, ...] = (), +) -> IncludeClosure: + """Resolve, validate, and hash the full recursive include dependency closure.""" + policy.resolve(root) + artifacts: dict[PurePosixPath, IncludedArtifact] = {} + edges: list[IncludeEdge] = [] + visited: set[PurePosixPath] = set() + + def visit(source: PurePosixPath, stack: tuple[PurePosixPath, ...]) -> None: + if source in stack: + cycle = " -> ".join(path.as_posix() for path in stack + (source,)) + raise IncludeGraphError(f"include cycle detected: {cycle}") + if source in visited: + return + visited.add(source) + resolved_source = policy.resolve(source) + try: + text = resolved_source.canonical_path.read_text(encoding="utf-8") + except UnicodeDecodeError as exc: + raise IncludeGraphError( + f"included artifact is not UTF-8 text: {source}" + ) from exc + for reference in parse_include_references(text): + targets = _resolved_targets( + source, + reference, + policy, + root_aliases if source == root else (), + ) + for target in targets: + try: + snapshot = policy.snapshot("include", target) + except (FileNotFoundError, PathPolicyError) as exc: + if reference.optional and isinstance(exc, FileNotFoundError): + edges.append(IncludeEdge(source, target, reference, False)) + continue + raise IncludeGraphError( + f"include cannot be resolved from {source}: {reference.path}" + ) from exc + if snapshot.digest is None or snapshot.git_mode is None: + if reference.optional: + edges.append(IncludeEdge(source, target, reference, False)) + continue + if _external_package_reference(reference.path): + edges.append(IncludeEdge(source, target, reference, False)) + continue + raise IncludeGraphError(f"required include is missing: {target}") + edges.append(IncludeEdge(source, target, reference, True)) + artifacts[target] = IncludedArtifact( + target, snapshot.digest, snapshot.git_mode + ) + visit(target, stack + (source,)) + + visit(root, ()) + return IncludeClosure( + root, + tuple(sorted(artifacts.values())), + tuple(sorted(edges)), + any( + edge.reference.query + or (not edge.target_exists and not edge.reference.optional) + for edge in edges + ), + ) diff --git a/pdd/sync_core/language.py b/pdd/sync_core/language.py new file mode 100644 index 000000000..6fc3445ab --- /dev/null +++ b/pdd/sync_core/language.py @@ -0,0 +1,182 @@ +"""Protected language and artifact-format identity registry.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Optional + + +class LanguageRegistryError(ValueError): + """Raised when language identity is unknown, ambiguous, or inconsistent.""" + + +_LEGACY_ALIASES = { + "c-plus-plus": ("cpp",), + "c-sharp": ("csharp",), + "f-sharp": ("fsharp",), + "javascriptreact": ("jsx",), + "objective-c": ("objectivec",), + "restructuredtext": ("rst",), + "typescriptreact": ("tsx",), + "yaml": ("yml",), +} + + +def _language_id(name: str) -> str: + expanded = name.casefold().replace("++", " plus plus ").replace("#", " sharp ") + identifier = re.sub(r"[^a-z0-9]+", "-", expanded).strip("-") + if not identifier: + raise LanguageRegistryError(f"language has no stable identifier: {name!r}") + return identifier + + +@dataclass(frozen=True, order=True) +class LanguageSpec: + """Stable language identity and its explicitly supported output formats.""" + + language_id: str + display_name: str + aliases: tuple[str, ...] + extensions: tuple[str, ...] + output_roles: tuple[str, ...] + + +@dataclass(frozen=True) +class _LanguageRow: + display_name: str + extension: str + output_roles: tuple[str, ...] + + +def _read_rows(path: Path) -> tuple[_LanguageRow, ...]: + rows: list[_LanguageRow] = [] + try: + with path.open(newline="", encoding="utf-8") as handle: + for raw in csv.DictReader(handle): + name = (raw.get("language") or "").strip() + if not name: + raise LanguageRegistryError("language registry contains an empty name") + extension = (raw.get("extension") or "").strip().casefold() + if extension and not extension.startswith("."): + extension = "." + extension + roles = tuple( + sorted( + role.strip().casefold() + for role in (raw.get("outputs") or "").split("|") + if role.strip() + ) + ) + rows.append(_LanguageRow(name, extension, roles)) + except (OSError, csv.Error) as exc: + raise LanguageRegistryError(f"cannot read language registry: {path}") from exc + return tuple(rows) + + +class LanguageRegistry: + """Deterministic alias and extension resolution with ambiguity rejection.""" + + def __init__(self, specs: Iterable[LanguageSpec]) -> None: + ordered = tuple(sorted(specs)) + if not ordered: + raise LanguageRegistryError("language registry must not be empty") + self.specs = ordered + self._by_id = {spec.language_id: spec for spec in ordered} + if len(self._by_id) != len(ordered): + raise LanguageRegistryError("duplicate stable language ID") + self._by_alias: dict[str, LanguageSpec] = {} + self._by_extension: dict[str, list[LanguageSpec]] = {} + for spec in ordered: + for alias in spec.aliases: + normalized = alias.casefold() + previous = self._by_alias.get(normalized) + if previous is not None and previous != spec: + raise LanguageRegistryError(f"ambiguous language alias: {alias}") + self._by_alias[normalized] = spec + for extension in spec.extensions: + self._by_extension.setdefault(extension, []).append(spec) + + @classmethod + def from_csv(cls, path: Path) -> "LanguageRegistry": + """Load and merge all rows without selecting a first-row winner.""" + grouped: dict[str, list[_LanguageRow]] = {} + for row in _read_rows(path): + grouped.setdefault(_language_id(row.display_name), []).append(row) + specs: list[LanguageSpec] = [] + for language_id, rows in grouped.items(): + displays = {row.display_name for row in rows} + aliases = tuple( + sorted( + {name.casefold() for name in displays} + | {language_id} + | set(_LEGACY_ALIASES.get(language_id, ())) + ) + ) + extensions = tuple(sorted({row.extension for row in rows})) + roles = tuple(sorted({role for row in rows for role in row.output_roles})) + specs.append( + LanguageSpec( + language_id, + sorted(displays, key=str.casefold)[0], + aliases, + extensions, + roles, + ) + ) + return cls(specs) + + @classmethod + def bundled(cls) -> "LanguageRegistry": + """Load the package-bundled registry in source and wheel installations.""" + return cls.from_csv(Path(__file__).parents[1] / "data" / "language_format.csv") + + def resolve_alias(self, alias: str) -> LanguageSpec: + """Resolve one explicit language alias to its stable identity.""" + spec = self._by_alias.get(alias.casefold()) + if spec is None: + raise LanguageRegistryError(f"unknown language alias: {alias}") + return spec + + def resolve_extension( + self, + extension: str, + *, + explicit_language: Optional[str] = None, + ) -> LanguageSpec: + """Resolve an extension only when it names exactly one language.""" + normalized = extension.casefold() + if normalized and not normalized.startswith("."): + normalized = "." + normalized + if explicit_language is not None: + spec = self.resolve_alias(explicit_language) + if normalized not in spec.extensions: + raise LanguageRegistryError( + f"extension {normalized!r} is not valid for {spec.language_id}" + ) + return spec + matches = self._by_extension.get(normalized, []) + if len(matches) != 1: + names = ", ".join(spec.language_id for spec in matches) or "none" + raise LanguageRegistryError( + f"extension {normalized!r} is ambiguous or unknown: {names}" + ) + return matches[0] + + def digest(self) -> str: + """Return the protected deterministic registry digest.""" + payload = [ + { + "language_id": spec.language_id, + "display_name": spec.display_name, + "aliases": spec.aliases, + "extensions": spec.extensions, + "output_roles": spec.output_roles, + } + for spec in self.specs + ] + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py new file mode 100644 index 000000000..6ca564049 --- /dev/null +++ b/pdd/sync_core/lifecycle.py @@ -0,0 +1,159 @@ +"""Credential-free execution of checker-owned packaged lifecycle scenarios.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +from .certificate import LifecycleResult +from .scenario_contract import REQUIRED_SCENARIOS + + +_SECRET_ENV_MARKERS = ( + "API_KEY", + "CREDENTIAL", + "PASSWORD", + "SECRET", + "SIGNING_KEY", + "TOKEN", +) + + +def _isolated_lifecycle_environment(home: Path) -> dict[str, str]: + """Build a credential-free environment with no source import overrides.""" + environment = { + key: value + for key, value in os.environ.items() + if not any(marker in key.upper() for marker in _SECRET_ENV_MARKERS) + and key not in {"PYTHONPATH", "PYTHONHOME", "PDD_PATH"} + } + environment["HOME"] = str(home) + environment["XDG_CONFIG_HOME"] = str(home / ".config") + environment["XDG_CACHE_HOME"] = str(home / ".cache") + environment["PYTHONNOUSERSITE"] = "1" + return environment + + +def _failed_result(*, timeout: bool = False) -> LifecycleResult: + return LifecycleResult( + len(REQUIRED_SCENARIOS), + 0, + 0, + int(timeout), + 1, + 1, + tuple(sorted(REQUIRED_SCENARIOS)), + ) + + +def _normalized_results(payload: Any) -> dict[str, dict[str, Any]]: + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise ValueError("released lifecycle result schema is invalid") + rows = payload.get("results") + if not isinstance(rows, list): + raise ValueError("released lifecycle results are absent") + results: dict[str, dict[str, Any]] = {} + for row in rows: + if not isinstance(row, dict): + raise ValueError("released lifecycle result is malformed") + scenario_id = row.get("scenario_id") + status = row.get("status") + if ( + not isinstance(scenario_id, str) + or scenario_id in results + or status not in {"PASS", "FAIL", "MISSING"} + ): + raise ValueError("released lifecycle result identity is invalid") + for metric in ( + "required_tests_skipped_or_xfailed", + "collection_errors", + "timeouts", + "post_repair_second_run_writes", + "post_merge_tree_changes", + ): + value = row.get(metric) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError("released lifecycle metric is invalid") + results[scenario_id] = row + if set(results) != set(REQUIRED_SCENARIOS): + raise ValueError("released lifecycle scenario set is incomplete") + return results + + +def run_lifecycle_matrix( + root: Path, + *, + cloud_root: Path | None = None, + cloud_base_ref: str | None = None, + cloud_head_ref: str | None = None, + timeout_seconds: int = 1200, +) -> LifecycleResult: + # pylint: disable=too-many-arguments + """Run only the scenario harness installed with the released checker.""" + del root # Candidate repository tests are never lifecycle evidence. + if cloud_root is None or cloud_base_ref is None or cloud_head_ref is None: + return _failed_result() + with tempfile.TemporaryDirectory(prefix="pdd-released-lifecycle-") as directory: + temporary = Path(directory) + output = temporary / "result.json" + home = temporary / "home" + home.mkdir(mode=0o700) + command = [ + sys.executable, + "-I", + "-m", + "pdd.sync_core.scenario_harness", + "--output", + str(output), + "--cloud-root", + str(Path(cloud_root).resolve()), + "--cloud-base-ref", + cloud_base_ref, + "--cloud-head-ref", + cloud_head_ref, + ] + try: + completed = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + timeout=timeout_seconds, + env=_isolated_lifecycle_environment(home), + ) + except subprocess.TimeoutExpired: + return _failed_result(timeout=True) + try: + results = _normalized_results(json.loads(output.read_text(encoding="utf-8"))) + except (OSError, ValueError, json.JSONDecodeError): + return _failed_result() + missing = tuple( + sorted( + scenario_id + for scenario_id, row in results.items() + if row["status"] == "MISSING" + ) + ) + failures = sum(row["status"] != "PASS" for row in results.values()) + if completed.returncode != 0 and failures == 0: + failures = 1 + return LifecycleResult( + failures, + sum( + int(row["required_tests_skipped_or_xfailed"]) + for row in results.values() + ), + sum(int(row["collection_errors"]) for row in results.values()), + sum(int(row["timeouts"]) for row in results.values()), + sum( + int(row["post_repair_second_run_writes"]) + for row in results.values() + ), + sum(int(row["post_merge_tree_changes"]) for row in results.values()), + missing, + ) diff --git a/pdd/sync_core/manifest.py b/pdd/sync_core/manifest.py new file mode 100644 index 000000000..1cfee29ab --- /dev/null +++ b/pdd/sync_core/manifest.py @@ -0,0 +1,929 @@ +"""Deterministic base/head candidate inventory and unit manifest.""" + +from __future__ import annotations + +import hashlib +import fnmatch +import json +import posixpath +import subprocess +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Optional + +import yaml + +from .identity import REPOSITORY_ID_RELPATH, canonical_repository_id +from .git_io import read_git_blob +from .language import LanguageRegistry, LanguageRegistryError +from .types import CandidateId, InventoryStatus, UnitId + + +class ManifestError(ValueError): + """Raised when Git inventory or protected manifest inputs are invalid.""" + + +@dataclass(frozen=True, order=True) +class GitTreeEntry: + """Mode and object identity for one tracked path in a Git tree.""" + + relpath: PurePosixPath + git_mode: str + object_id: str + + +@dataclass(frozen=True, order=True) +# pylint: disable=too-many-instance-attributes +class CandidateRecord: + """Complete base/head accounting for one tracked candidate path.""" + + candidate_id: CandidateId + inventory: InventoryStatus + in_base: bool + in_head: bool + ownership_provenance: str + base_object_id: str | None = None + base_git_mode: str | None = None + head_object_id: str | None = None + head_git_mode: str | None = None + unit_id: UnitId | None = None + + +@dataclass(frozen=True, order=True) +class ManifestUnit: + """Prompt-backed unit identity and its exact architecture-owned outputs.""" + + unit_id: UnitId + present_in_base: bool + present_in_head: bool + artifact_paths: tuple[PurePosixPath, ...] + tombstoned: bool + + +@dataclass(frozen=True) +class ManifestRefs: + """Protected base and checked-head Git references.""" + + base: str + head: str + + +@dataclass(frozen=True) +class DecommissionTombstone: + """Protected proof that a synchronized unit was deliberately removed.""" + + prompt_path: PurePosixPath + artifact_paths: tuple[PurePosixPath, ...] + rationale: str + owner: str + baseline_status: str + + +@dataclass(frozen=True, order=True) +class OwnershipRule: + """Protected-base classification for an otherwise unmatched tracked path.""" + + pattern: str + inventory: InventoryStatus + role: str + owner: str + + +@dataclass(frozen=True) +class _TreeManifest: + """Parsed candidate inputs for one immutable Git tree.""" + + ref: str + entries: dict[PurePosixPath, GitTreeEntry] + units: dict[PurePosixPath, UnitId] + outputs: dict[PurePosixPath, UnitId] + invalid_reasons: tuple[str, ...] + + +@dataclass(frozen=True) +class _UnitSources: + """Base/head identity and output maps used during unit assembly.""" + + base_units: dict[PurePosixPath, UnitId] + head_units: dict[PurePosixPath, UnitId] + base_outputs: dict[PurePosixPath, UnitId] + head_outputs: dict[PurePosixPath, UnitId] + + +@dataclass(frozen=True) +class _CandidateSources: + """Inputs used to classify the independent tracked-path partition.""" + + repository_id: str + base_entries: dict[PurePosixPath, GitTreeEntry] + head_entries: dict[PurePosixPath, GitTreeEntry] + prompt_owner: dict[PurePosixPath, UnitId] + output_owner: dict[PurePosixPath, UnitId] + ownership_rules: tuple[OwnershipRule, ...] + + +@dataclass(frozen=True) +class UnitManifest: + """Deterministic partition of the protected base/head candidate universe.""" + + repository_id: str + language_registry_digest: str + refs: ManifestRefs + candidates: tuple[CandidateRecord, ...] + units: tuple[ManifestUnit, ...] + expected_managed: tuple[UnitId, ...] + invalid_reasons: tuple[str, ...] + unaccounted_tracked_paths: tuple[PurePosixPath, ...] + + @property + def base_ref(self) -> str: + """Return the protected base reference.""" + return self.refs.base + + @property + def head_ref(self) -> str: + """Return the checked head reference.""" + return self.refs.head + + @property + def managed_units(self) -> tuple[ManifestUnit, ...]: + """Return prompt-backed units currently present in the head tree.""" + return tuple(unit for unit in self.units if unit.present_in_head) + + def digest(self) -> str: + """Return a deterministic digest of identity, ownership, and paths.""" + structural_candidates = [ + item + for item in self.candidates + if not _is_dynamic_canonical_state(item.candidate_id.artifact_relpath) + ] + payload = { + "repository_id": self.repository_id, + "language_registry_digest": self.language_registry_digest, + "candidates": [ + { + "path": item.candidate_id.artifact_relpath.as_posix(), + "role": item.candidate_id.role, + "inventory": item.inventory.value, + "in_base": item.in_base, + "in_head": item.in_head, + "provenance": item.ownership_provenance, + "base_object_id": item.base_object_id, + "base_git_mode": item.base_git_mode, + "head_object_id": item.head_object_id, + "head_git_mode": item.head_git_mode, + "unit": _unit_payload(item.unit_id), + } + for item in structural_candidates + ], + "units": [ + { + "id": _unit_payload(item.unit_id), + "present_in_base": item.present_in_base, + "present_in_head": item.present_in_head, + "artifact_paths": [path.as_posix() for path in item.artifact_paths], + "tombstoned": item.tombstoned, + } + for item in self.units + ], + "expected_managed": [_unit_payload(item) for item in self.expected_managed], + "invalid_reasons": self.invalid_reasons, + "unaccounted": [path.as_posix() for path in self.unaccounted_tracked_paths], + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _unit_payload(unit_id: UnitId | None) -> dict[str, str] | None: + if unit_id is None: + return None + return { + "repository_id": unit_id.repository_id, + "prompt_relpath": unit_id.prompt_relpath.as_posix(), + "language_id": unit_id.language_id, + } + + +def _git(root: Path, *args: str) -> bytes: + result = subprocess.run( + ["git", *args], + cwd=root, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = result.stderr.decode(errors="replace").strip() + raise ManifestError(f"Git inventory command failed: {detail}") + return result.stdout + + +def _tree_entries(root: Path, ref: str) -> dict[PurePosixPath, GitTreeEntry]: + raw = _git(root, "ls-tree", "-r", "-z", "--full-tree", ref) + entries: dict[PurePosixPath, GitTreeEntry] = {} + for record in raw.split(b"\0"): + if not record: + continue + metadata, path_bytes = record.split(b"\t", 1) + mode, object_type, object_id = metadata.decode("ascii").split() + if object_type not in {"blob", "commit"}: + continue + relpath = PurePosixPath(path_bytes.decode("utf-8", errors="strict")) + entries[relpath] = GitTreeEntry(relpath, mode, object_id) + return entries + + +def _blob(root: Path, ref: str, path: PurePosixPath) -> bytes: + return _git(root, "show", f"{ref}:{path.as_posix()}") + + +def _prompt_units( + ref: str, + entries: dict[PurePosixPath, GitTreeEntry], + repository_id: str, + registry: LanguageRegistry, + ownership_rules: tuple[OwnershipRule, ...], + protected_owned_paths: set[PurePosixPath], +) -> tuple[dict[PurePosixPath, UnitId], list[str]]: + # pylint: disable=too-many-arguments,too-many-positional-arguments + units: dict[PurePosixPath, UnitId] = {} + invalid: list[str] = [] + for path in sorted(entries): + if path.suffix != ".prompt" or "_" not in path.stem: + continue + rule, rule_error = ( + _ownership_for(path, ownership_rules) + if path in protected_owned_paths + else (None, None) + ) + if rule_error: + invalid.append(f"{ref}:{rule_error}") + continue + if rule is not None and rule.inventory is InventoryStatus.HUMAN_OWNED: + continue + language_alias = path.stem.rsplit("_", 1)[1] + try: + language = registry.resolve_alias(language_alias) + except LanguageRegistryError as exc: + invalid.append(f"{ref}:{path.as_posix()}: {exc}") + continue + units[path] = UnitId(repository_id, path, language.language_id) + return units, invalid + + +def _architecture_outputs( + root: Path, + ref: str, + entries: dict[PurePosixPath, GitTreeEntry], + prompt_units: dict[PurePosixPath, UnitId], + ownership_rules: tuple[OwnershipRule, ...], + protected_owned_paths: set[PurePosixPath], +) -> tuple[dict[PurePosixPath, UnitId], list[str]]: + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals + outputs: dict[PurePosixPath, UnitId] = {} + invalid: list[str] = [] + by_name = _paths_by_name(prompt_units) + for architecture_path in sorted( + path for path in entries if path.name == "architecture.json" + ): + rule, rule_error = ( + _ownership_for(architecture_path, ownership_rules) + if architecture_path in protected_owned_paths + else (None, None) + ) + if rule_error: + invalid.append(f"{ref}:{rule_error}") + continue + if rule is not None and rule.role == "excluded-project": + continue + modules, error = _architecture_modules(root, ref, architecture_path) + if error: + invalid.append(error) + continue + mapped, mapping_errors = _map_architecture_modules( + ref, architecture_path, modules, by_name, prompt_units + ) + invalid.extend(mapping_errors) + for output, unit_id in mapped.items(): + previous = outputs.get(output) + if previous is not None and previous != unit_id: + invalid.append(f"{ref}:{output.as_posix()}: duplicate unit ownership") + else: + outputs[output] = unit_id + return outputs, invalid + + +def _governing_config( + prompt_path: PurePosixPath, + entries: dict[PurePosixPath, GitTreeEntry], +) -> PurePosixPath | None: + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals + for parent in (prompt_path.parent, *prompt_path.parents): + candidate = parent / ".pddrc" + if candidate in entries: + return candidate + root = PurePosixPath(".pddrc") + return root if root in entries else None + + +def _config_context( + payload: object, + config_path: PurePosixPath, + prompt_path: PurePosixPath, +) -> tuple[dict[str, object] | None, PurePosixPath | None, str | None]: + contexts = payload.get("contexts") if isinstance(payload, dict) else None + if not isinstance(contexts, dict): + return None, None, "contexts must be a mapping" + matches: list[tuple[int, int, dict[str, object], PurePosixPath]] = [] + for index, (_name, context) in enumerate(contexts.items()): + defaults = context.get("defaults") if isinstance(context, dict) else None + prompts_dir = defaults.get("prompts_dir") if isinstance(defaults, dict) else None + if not isinstance(prompts_dir, str) or not prompts_dir.strip(): + continue + root = config_path.parent / PurePosixPath(prompts_dir) + root = PurePosixPath(posixpath.normpath(root.as_posix())) + try: + prompt_path.relative_to(root) + except ValueError: + continue + matches.append((len(root.parts), index, defaults, root)) + if not matches: + return None, None, None + matches.sort(key=lambda item: (-item[0], item[1])) + return matches[0][2], matches[0][3], None + + +def _configured_output( + prompt_path: PurePosixPath, + unit_id: UnitId, + config_path: PurePosixPath, + defaults: dict[str, object], + prompts_root: PurePosixPath, + registry: LanguageRegistry, +) -> PurePosixPath | None: + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals + relative = prompt_path.relative_to(prompts_root) + name = relative.stem.rsplit("_", 1)[0] + category = "" if relative.parent == PurePosixPath(".") else relative.parent.as_posix() + values = { + "name": name, + "language": unit_id.language_id, + "category": category, + "dir_prefix": f"{category}/" if category else "", + } + outputs = defaults.get("outputs") + code = outputs.get("code") if isinstance(outputs, dict) else None + template = code.get("path") if isinstance(code, dict) else None + if isinstance(template, str) and template: + try: + rendered = template.format_map(values) + except (KeyError, ValueError) as exc: + raise ManifestError("configured code output template is invalid") from exc + return config_path.parent / PurePosixPath(rendered) + generate = defaults.get("generate_output_path") + if not isinstance(generate, str) or not generate: + return None + spec = registry.resolve_alias(unit_id.language_id) + extensions = tuple(item for item in spec.extensions if item) + if len(extensions) != 1: + raise ManifestError("configured output language has no unique extension") + output_root = config_path.parent / PurePosixPath(generate) + return output_root / relative.parent / f"{name}{extensions[0]}" + + +def _config_outputs( + root: Path, + ref: str, + entries: dict[PurePosixPath, GitTreeEntry], + prompt_units: dict[PurePosixPath, UnitId], + architecture_outputs: dict[PurePosixPath, UnitId], + registry: LanguageRegistry, +) -> tuple[dict[PurePosixPath, UnitId], list[str]]: + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals + outputs: dict[PurePosixPath, UnitId] = {} + invalid: list[str] = [] + architecture_units = set(architecture_outputs.values()) + configs: dict[PurePosixPath, object] = {} + for prompt_path, unit_id in prompt_units.items(): + if unit_id in architecture_units: + continue + config_path = _governing_config(prompt_path, entries) + if config_path is None: + continue + if config_path not in configs: + try: + configs[config_path] = yaml.safe_load(_blob(root, ref, config_path)) + except (UnicodeDecodeError, yaml.YAMLError) as exc: + invalid.append(f"{ref}:{config_path}: invalid config: {exc}") + configs[config_path] = None + defaults, prompts_root, error = _config_context( + configs[config_path], config_path, prompt_path + ) + if error: + invalid.append(f"{ref}:{prompt_path}: {error}") + continue + if defaults is None or prompts_root is None: + continue + try: + output = _configured_output( + prompt_path, + unit_id, + config_path, + defaults, + prompts_root, + registry, + ) + except (LanguageRegistryError, ManifestError) as exc: + invalid.append(f"{ref}:{prompt_path}: {exc}") + continue + if output is None: + continue + output = PurePosixPath(posixpath.normpath(output.as_posix())) + if output.is_absolute() or ".." in output.parts: + invalid.append(f"{ref}:{prompt_path}: configured output escapes repository") + continue + previous = outputs.get(output) or architecture_outputs.get(output) + if previous is not None and previous != unit_id: + invalid.append(f"{ref}:{output}: duplicate configured output ownership") + continue + outputs[output] = unit_id + return outputs, invalid + + +def _paths_by_name( + prompt_units: dict[PurePosixPath, UnitId], +) -> dict[str, list[PurePosixPath]]: + """Index prompt identities without resolving ambiguous leaves.""" + by_name: dict[str, list[PurePosixPath]] = {} + for path in prompt_units: + by_name.setdefault(path.name, []).append(path) + return by_name + + +def _architecture_modules( + root: Path, + ref: str, + architecture_path: PurePosixPath, +) -> tuple[list[object], str | None]: + """Load one architecture module list from an immutable Git blob.""" + try: + payload = json.loads(_blob(root, ref, architecture_path)) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + return [], f"{ref}:{architecture_path.as_posix()}: invalid JSON: {exc}" + modules = payload.get("modules", []) if isinstance(payload, dict) else payload + if not isinstance(modules, list): + return [], f"{ref}:{architecture_path.as_posix()}: modules must be a list" + return modules, None + + +def _map_architecture_modules( + ref: str, + architecture_path: PurePosixPath, + modules: list[object], + by_name: dict[str, list[PurePosixPath]], + prompt_units: dict[PurePosixPath, UnitId], +) -> tuple[dict[PurePosixPath, UnitId], list[str]]: + """Map exact architecture entries without basename guessing.""" + # pylint: disable=too-many-locals + outputs: dict[PurePosixPath, UnitId] = {} + invalid: list[str] = [] + for item in modules: + if not isinstance(item, dict): + invalid.append(f"{ref}:{architecture_path.as_posix()}: invalid module entry") + continue + filename = item.get("filename") + filepath = item.get("filepath") + if not isinstance(filename, str) or not isinstance(filepath, str): + continue + if PurePosixPath(filename).suffix != ".prompt": + continue + declared = PurePosixPath(filename) + parent = architecture_path.parent + exact_candidates = ( + parent / "prompts" / declared, + parent / "pdd" / "prompts" / declared, + parent / declared, + PurePosixPath("prompts") / declared, + ) + exact_match = next( + (path for path in exact_candidates if path in prompt_units), None + ) + matches = [exact_match] if exact_match is not None else by_name.get( + declared.name, [] + ) + if len(matches) != 1: + invalid.append( + f"{ref}:{architecture_path.as_posix()}: prompt mapping for " + f"{filename!r} has {len(matches)} matches" + ) + continue + declared_output = PurePosixPath(filepath) + output = parent / declared_output + if declared_output.is_absolute() or ".." in declared_output.parts: + invalid.append(f"{ref}:{architecture_path.as_posix()}: invalid output {filepath}") + continue + if output in outputs: + invalid.append( + f"{ref}:{architecture_path.as_posix()}: duplicate output {filepath}" + ) + continue + outputs[output] = prompt_units[matches[0]] + return outputs, invalid + + +def _tombstones( + root: Path, + ref: str, + entries: dict[PurePosixPath, GitTreeEntry], +) -> dict[PurePosixPath, DecommissionTombstone]: + path = PurePosixPath(".pdd/sync-tombstones.json") + if path not in entries: + return {} + try: + payload = json.loads(_blob(root, ref, path)) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ManifestError("protected sync tombstones are malformed") from exc + if not isinstance(payload, list): + raise ManifestError("protected sync tombstones must be a list") + tombstones: dict[PurePosixPath, DecommissionTombstone] = {} + for item in payload: + if not isinstance(item, dict): + raise ManifestError("each sync tombstone must be an object") + try: + prompt_path = PurePosixPath(item["prompt_path"]) + artifacts = tuple(sorted(PurePosixPath(value) for value in item["artifact_paths"])) + tombstone = DecommissionTombstone( + prompt_path, + artifacts, + item["rationale"], + item["owner"], + item["baseline_status"], + ) + except (KeyError, TypeError) as exc: + raise ManifestError("sync tombstone is missing required fields") from exc + if ( + prompt_path.is_absolute() + or ".." in prompt_path.parts + or any(path.is_absolute() or ".." in path.parts for path in artifacts) + or not tombstone.rationale + or not tombstone.owner + ): + raise ManifestError("sync tombstone contains invalid protected fields") + if prompt_path in tombstones: + raise ManifestError("duplicate sync tombstone prompt identity") + tombstones[prompt_path] = tombstone + return tombstones + + +def _manifest_unit( + prompt_path: PurePosixPath, + sources: _UnitSources, + tombstone: DecommissionTombstone | None, + head_ref: str, +) -> tuple[ManifestUnit, str | None]: + """Build one unit and validate any decommission transition.""" + unit_id = sources.head_units.get(prompt_path) or sources.base_units[prompt_path] + base_artifacts = { + path for path, owner in sources.base_outputs.items() if owner == unit_id + } + head_artifacts = { + path for path, owner in sources.head_outputs.items() if owner == unit_id + } + removed = prompt_path in sources.base_units and prompt_path not in sources.head_units + tombstoned = bool( + removed + and tombstone + and set(tombstone.artifact_paths) == base_artifacts | {prompt_path} + and tombstone.baseline_status == "IN_SYNC" + ) + reason = None + if removed and not tombstoned: + reason = ( + f"{head_ref}:{prompt_path.as_posix()}: removed managed unit lacks " + "a complete IN_SYNC tombstone" + ) + elif tombstone and not removed: + reason = f"{head_ref}:{prompt_path.as_posix()}: tombstone targets a present unit" + unit = ManifestUnit( + unit_id, + prompt_path in sources.base_units, + prompt_path in sources.head_units, + tuple(sorted(base_artifacts | head_artifacts)), + tombstoned, + ) + return unit, reason + + +def _manifest_units( + sources: _UnitSources, + tombstones: dict[PurePosixPath, DecommissionTombstone], + head_ref: str, +) -> tuple[list[ManifestUnit], list[str]]: + """Assemble units and validate protected decommission transitions.""" + units: list[ManifestUnit] = [] + invalid: list[str] = [] + paths = set(sources.base_units) | set(sources.head_units) + for prompt_path in sorted(paths): + unit, reason = _manifest_unit( + prompt_path, sources, tombstones.get(prompt_path), head_ref + ) + units.append(unit) + if reason: + invalid.append(reason) + unknown = set(tombstones) - set(sources.base_units) + invalid.extend( + f"{head_ref}:{path.as_posix()}: tombstone has no base unit" + for path in sorted(unknown) + ) + return units, invalid + + +def _candidate_records( + sources: _CandidateSources, +) -> tuple[list[CandidateRecord], set[PurePosixPath], list[str]]: + """Partition every tracked path into managed or historical human ownership.""" + candidates: list[CandidateRecord] = [] + accounted: set[PurePosixPath] = set() + invalid: list[str] = [] + for path in sorted(set(sources.base_entries) | set(sources.head_entries)): + unit_id = sources.prompt_owner.get(path) or sources.output_owner.get(path) + rule, rule_error = ( + _ownership_for(path, sources.ownership_rules) + if path in sources.base_entries + else (None, None) + ) + if path in sources.prompt_owner: + role = "prompt" + inventory = InventoryStatus.MANAGED + provenance = "prompt-backed" + elif path in sources.output_owner: + role, inventory, provenance = "code", InventoryStatus.MANAGED, "architecture" + elif rule is not None and rule.role == "excluded-project": + role = rule.role + inventory = rule.inventory + provenance = f"protected-ownership:{rule.owner}:{rule.pattern}" + elif _is_protected_control(path): + role = "policy" + inventory = InventoryStatus.MANAGED + provenance = "protected-control" + else: + if rule is None: + role = "unaccounted" + inventory = InventoryStatus.INVALID + provenance = "none" + invalid.append( + rule_error or f"{path.as_posix()}: tracked path has no ownership rule" + ) + else: + role = rule.role + inventory = rule.inventory + provenance = f"protected-ownership:{rule.owner}:{rule.pattern}" + candidates.append( + CandidateRecord( + CandidateId(sources.repository_id, path, role), + inventory, + path in sources.base_entries, + path in sources.head_entries, + provenance, + sources.base_entries.get(path).object_id + if path in sources.base_entries + else None, + sources.base_entries.get(path).git_mode + if path in sources.base_entries + else None, + sources.head_entries.get(path).object_id + if path in sources.head_entries + else None, + sources.head_entries.get(path).git_mode + if path in sources.head_entries + else None, + unit_id, + ) + ) + if inventory is not InventoryStatus.INVALID: + accounted.add(path) + return candidates, accounted, invalid + + +def _is_protected_control(path: PurePosixPath) -> bool: + """Return whether a path is an intrinsic canonical policy/config input.""" + under_canonical_state = _is_dynamic_canonical_state(path) + return under_canonical_state or ( + path.name in {".pddrc", "architecture.json"} + or path + in { + PurePosixPath(".pdd/repository-id"), + PurePosixPath(".pdd/verification-profiles.json"), + PurePosixPath(".pdd/attestation-trust.json"), + PurePosixPath(".pdd/sync-ownership.json"), + PurePosixPath(".pdd/sync-tombstones.json"), + PurePosixPath(".pdd/sync-waivers.json"), + } + ) + + +def _is_dynamic_canonical_state(path: PurePosixPath) -> bool: + """Return whether content-addressed runtime state must not self-hash.""" + return path.parts[:3] in { + (".pdd", "meta", "v2"), + (".pdd", "evidence", "v2"), + } + + +def _ownership_for( + path: PurePosixPath, + rules: tuple[OwnershipRule, ...], +) -> tuple[OwnershipRule | None, str | None]: + matches = [rule for rule in rules if fnmatch.fnmatchcase(path.as_posix(), rule.pattern)] + if not matches: + return None, None + outcomes = {(item.inventory, item.role, item.owner) for item in matches} + if len(outcomes) != 1: + return None, f"{path.as_posix()}: protected ownership rules are ambiguous" + return matches[0], None + + +def _ownership_rules(root: Path, protected_base_ref: str) -> tuple[OwnershipRule, ...]: + """Load unmatched-path ownership only from the protected base tree.""" + path = PurePosixPath(".pdd/sync-ownership.json") + raw = read_git_blob(root, protected_base_ref, path) + if raw is None: + return () + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ManifestError("protected sync ownership policy is malformed") from exc + rows = payload.get("rules") if isinstance(payload, dict) else None + if not isinstance(rows, list): + raise ManifestError("protected sync ownership rules must be a list") + rules: list[OwnershipRule] = [] + for item in rows: + if not isinstance(item, dict): + raise ManifestError("protected ownership rule must be an object") + try: + pattern = str(item["pattern"]) + inventory = InventoryStatus(str(item["inventory"])) + role = str(item["role"]) + owner = str(item["owner"]) + except (KeyError, ValueError) as exc: + raise ManifestError("protected ownership rule is malformed") from exc + rule = OwnershipRule(pattern, inventory, role, owner) + if not _valid_ownership_rule(rule): + raise ManifestError("protected ownership rule is overly broad or invalid") + rules.append(rule) + return tuple(sorted(rules)) + + +def _valid_ownership_rule(rule: OwnershipRule) -> bool: + """Reject catch-all or escaping rules that could hide future managed debt.""" + path = PurePosixPath(rule.pattern) + pattern_valid = ( + rule.pattern not in {"*", "**", "**/*"} + and not rule.pattern.startswith("/") + and ".." not in path.parts + ) + identity_valid = bool(rule.role and rule.owner) + inventory_valid = rule.inventory in { + InventoryStatus.MANAGED, + InventoryStatus.HUMAN_OWNED, + } + return pattern_valid and identity_valid and inventory_valid + + +def _tree_manifest( + root: Path, + ref: str, + repository_id: str, + registry: LanguageRegistry, + ownership_rules: tuple[OwnershipRule, ...], + protected_owned_paths: Optional[set[PurePosixPath]] = None, +) -> _TreeManifest: + # pylint: disable=too-many-arguments,too-many-positional-arguments + """Parse one immutable tree into canonical units and architecture outputs.""" + entries = _tree_entries(root, ref) + units, unit_invalid = _prompt_units( + ref, + entries, + repository_id, + registry, + ownership_rules, + protected_owned_paths or set(entries), + ) + outputs, architecture_invalid = _architecture_outputs( + root, + ref, + entries, + units, + ownership_rules, + protected_owned_paths or set(entries), + ) + configured_outputs, config_invalid = _config_outputs( + root, + ref, + entries, + units, + outputs, + registry, + ) + outputs.update(configured_outputs) + generated_prompt_outputs = { + path + for path, owner in outputs.items() + if path in units and units[path] != owner + } + units = { + path: unit_id + for path, unit_id in units.items() + if path not in generated_prompt_outputs + } + return _TreeManifest( + ref, + entries, + units, + outputs, + tuple(unit_invalid + architecture_invalid + config_invalid), + ) + + +def _assemble_manifest( + repository_id: str, + language_registry_digest: str, + base: _TreeManifest, + head: _TreeManifest, + tombstones: dict[PurePosixPath, DecommissionTombstone], + ownership_rules: tuple[OwnershipRule, ...], +) -> UnitManifest: + # pylint: disable=too-many-arguments,too-many-positional-arguments + """Combine parsed trees into the final candidate and unit partition.""" + invalid = list(base.invalid_reasons + head.invalid_reasons) + sources = _UnitSources(base.units, head.units, base.outputs, head.outputs) + units, tombstone_invalid = _manifest_units(sources, tombstones, head.ref) + invalid.extend(tombstone_invalid) + all_paths = set(base.entries) | set(head.entries) + candidates, accounted, ownership_invalid = _candidate_records( + _CandidateSources( + repository_id, + base.entries, + head.entries, + {**base.units, **head.units}, + {**base.outputs, **head.outputs}, + ownership_rules, + ) + ) + invalid.extend(ownership_invalid) + return UnitManifest( + repository_id, + language_registry_digest, + ManifestRefs(base.ref, head.ref), + tuple(candidates), + tuple(sorted(units)), + tuple(sorted(unit.unit_id for unit in units)), + tuple(sorted(invalid)), + tuple(sorted(all_paths - accounted)), + ) + + +def build_unit_manifest( + root: Path, + *, + base_ref: str, + head_ref: str, + registry: Optional[LanguageRegistry] = None, +) -> UnitManifest: + # pylint: disable=too-many-locals + """Build the complete protected base/head candidate union from Git objects.""" + repository_root = Path(root).resolve() + identity_path = PurePosixPath(REPOSITORY_ID_RELPATH.as_posix()) + base_identity = read_git_blob(repository_root, base_ref, identity_path) + head_identity = read_git_blob(repository_root, head_ref, identity_path) + if base_identity is None or head_identity is None: + raise ManifestError("base and head must contain .pdd/repository-id") + try: + base_repository_id = canonical_repository_id(base_identity.decode("ascii")) + head_repository_id = canonical_repository_id(head_identity.decode("ascii")) + except (UnicodeDecodeError, ValueError) as exc: + raise ManifestError("protected repository identity is malformed") from exc + if base_repository_id != head_repository_id: + raise ManifestError("repository identity changed between protected base and head") + repository_id = base_repository_id + language_registry = registry or LanguageRegistry.bundled() + ownership = _ownership_rules(repository_root, base_ref) + base = _tree_manifest( + repository_root, base_ref, repository_id, language_registry, ownership + ) + head = _tree_manifest( + repository_root, + head_ref, + repository_id, + language_registry, + ownership, + set(base.entries), + ) + tombstones = _tombstones(repository_root, head_ref, head.entries) + return _assemble_manifest( + repository_id, + language_registry.digest(), + base, + head, + tombstones, + ownership, + ) diff --git a/pdd/sync_core/path_policy.py b/pdd/sync_core/path_policy.py new file mode 100644 index 000000000..071d96441 --- /dev/null +++ b/pdd/sync_core/path_policy.py @@ -0,0 +1,118 @@ +"""Protected repository path validation and artifact snapshots.""" + +from __future__ import annotations + +import hashlib +import os +import stat +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Mapping + +from .types import ArtifactSnapshot + + +class PathPolicyError(ValueError): + """Raised when a managed path violates protected repository policy.""" + + +@dataclass(frozen=True) +class ResolvedPath: + """Logical identity and prevalidated canonical target for a managed path.""" + + logical_relpath: PurePosixPath + canonical_path: Path + alias_relpath: PurePosixPath | None = None + + +class PathPolicy: + """Validate managed paths against one canonical checkout boundary.""" + + def __init__( + self, + checkout_root: Path, + approved_aliases: Mapping[PurePosixPath, PurePosixPath] | None = None, + ) -> None: + root = Path(checkout_root) + if root.is_symlink() or not root.is_dir(): + raise PathPolicyError("checkout root must be a real directory") + self.checkout_root = root.resolve() + self.approved_aliases = dict(approved_aliases or {}) + + @staticmethod + def _validate_relpath(relpath: PurePosixPath) -> None: + if relpath.is_absolute() or not relpath.parts or ".." in relpath.parts: + raise PathPolicyError(f"managed path must be repository-relative: {relpath}") + + def _within_root(self, path: Path) -> bool: + try: + path.relative_to(self.checkout_root) + return True + except ValueError: + return False + + def resolve(self, relpath: PurePosixPath, *, allow_missing: bool = False) -> ResolvedPath: + """Resolve a logical path without permitting unapproved link traversal.""" + self._validate_relpath(relpath) + candidate = self.checkout_root.joinpath(*relpath.parts) + alias_relpath = None + parts = relpath.parts + for index in range(1, len(parts) + 1): + logical_component = PurePosixPath(*parts[:index]) + component = self.checkout_root.joinpath(*parts[:index]) + if not component.exists() and not component.is_symlink(): + break + mode = component.lstat().st_mode + if stat.S_ISLNK(mode): + approved_target = self.approved_aliases.get(logical_component) + if approved_target is None: + raise PathPolicyError(f"unapproved managed symlink: {logical_component}") + self._validate_relpath(approved_target) + target = component.resolve(strict=True) + expected = self.checkout_root.joinpath(*approved_target.parts).resolve(strict=True) + if target != expected or not self._within_root(target): + raise PathPolicyError( + f"approved alias target changed: {logical_component}" + ) + alias_relpath = logical_component + canonical = candidate.resolve(strict=not allow_missing) + if not self._within_root(canonical): + raise PathPolicyError(f"managed path escapes checkout: {relpath}") + if canonical.exists(): + mode = canonical.stat().st_mode + if not stat.S_ISREG(mode): + raise PathPolicyError( + f"managed artifact is not a regular file: {relpath}" + ) + elif not allow_missing: + raise PathPolicyError(f"managed artifact does not exist: {relpath}") + return ResolvedPath(relpath, canonical, alias_relpath) + + def snapshot( + self, + role: str, + relpath: PurePosixPath, + *, + required: bool = True, + ) -> ArtifactSnapshot: + """Hash content and normalized Git mode after validating path policy.""" + try: + resolved = self.resolve(relpath) + except FileNotFoundError: + return ArtifactSnapshot(role, relpath, None, None, required) + digest = hashlib.sha256(resolved.canonical_path.read_bytes()).hexdigest() + file_mode = resolved.canonical_path.stat().st_mode + executable_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + git_mode = "100755" if file_mode & executable_bits else "100644" + return ArtifactSnapshot(role, relpath, digest, git_mode, required) + + def revalidate(self, resolved: ResolvedPath) -> None: + """Reject target replacement or alias retarget before a later commit.""" + current = self.resolve(resolved.logical_relpath, allow_missing=True) + if current != resolved: + raise PathPolicyError( + f"managed path changed after planning: {resolved.logical_relpath}" + ) + parent_mode = os.lstat(current.canonical_path.parent).st_mode + if not stat.S_ISDIR(parent_mode): + raise PathPolicyError("managed destination parent is not a directory") diff --git a/pdd/sync_core/planner.py b/pdd/sync_core/planner.py new file mode 100644 index 000000000..1142b1133 --- /dev/null +++ b/pdd/sync_core/planner.py @@ -0,0 +1,134 @@ +"""Policy-preserving repair plans derived from canonical verdicts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from .types import BaselineStatus, InventoryStatus, SemanticStatus, SyncVerdict + + +class RepairAction(str, Enum): + """Explicit operation that can move a unit toward synchronization.""" + + NONE = "NONE" + VALIDATE = "VALIDATE" + SYNC_FROM_PROMPT = "SYNC_FROM_PROMPT" + UPDATE_PROMPT_REVIEW = "UPDATE_PROMPT_REVIEW" + RESOLVE_CONFLICT = "RESOLVE_CONFLICT" + VERIFIED_REBUILD = "VERIFIED_REBUILD" + BASELINE_RESET = "BASELINE_RESET" + RESTORE_REQUIRED = "RESTORE_REQUIRED" + REPAIR_VALIDATION = "REPAIR_VALIDATION" + BLOCK = "BLOCK" + + +@dataclass(frozen=True) +class RepairPolicy: + """Protected controls for the small set of unattended repair operations.""" + + allow_prompt_side_generation: bool = True + allow_verified_rebuild: bool = False + + +@dataclass(frozen=True) +class RepairPlan: + """Allowed action without changing the classifier's source verdict.""" + + verdict: SyncVerdict + action: RepairAction + unattended: bool + review_required: bool + reason: str + + +_PROMPT_ROLES = frozenset({"prompt", "include", "config", "architecture", "manifest"}) + + +def _preflight_plan( + verdict: SyncVerdict, policy: RepairPolicy +) -> RepairPlan | None: + """Plan terminal inventory, corruption, missing, and baseline conditions.""" + if verdict.in_sync: + return RepairPlan(verdict, RepairAction.NONE, False, False, "already in sync") + if ( + verdict.inventory is InventoryStatus.INVALID + or verdict.baseline is BaselineStatus.CORRUPT + ): + return RepairPlan( + verdict, RepairAction.BLOCK, False, True, "invalid or corrupt state blocks repair" + ) + if verdict.required_artifacts_missing: + return RepairPlan( + verdict, + RepairAction.RESTORE_REQUIRED, + False, + True, + "required artifacts must be restored or regenerated without stamping", + ) + if verdict.baseline is BaselineStatus.UNBASELINED: + action = ( + RepairAction.VERIFIED_REBUILD + if policy.allow_verified_rebuild + else RepairAction.BASELINE_RESET + ) + return RepairPlan( + verdict, + action, + False, + True, + "unbaselined state requires an audited reviewed transition", + ) + return None + + +def _semantic_plan(verdict: SyncVerdict) -> RepairPlan | None: + """Plan conflict, failed validation, and evidence-only transitions.""" + if verdict.semantic is SemanticStatus.CONFLICT: + return RepairPlan( + verdict, + RepairAction.RESOLVE_CONFLICT, + False, + True, + "simultaneous source and derived edits require explicit resolution", + ) + if verdict.semantic is SemanticStatus.FAILED: + return RepairPlan( + verdict, + RepairAction.REPAIR_VALIDATION, + False, + True, + "failed validation cannot be accepted as a baseline", + ) + if verdict.baseline is BaselineStatus.CURRENT: + return RepairPlan( + verdict, + RepairAction.VALIDATE, + True, + False, + "current bytes require complete trusted semantic evidence", + ) + return None + + +def plan_repair(verdict: SyncVerdict, policy: RepairPolicy) -> RepairPlan: + """Map a verdict to one safe action without accepting drift as synchronization.""" + plan = _preflight_plan(verdict, policy) or _semantic_plan(verdict) + if plan is not None: + return plan + changed = set(verdict.changed_roles) + if changed and changed.issubset(_PROMPT_ROLES): + return RepairPlan( + verdict, + RepairAction.SYNC_FROM_PROMPT, + policy.allow_prompt_side_generation, + not policy.allow_prompt_side_generation, + "prompt-side drift may regenerate derived artifacts under policy", + ) + return RepairPlan( + verdict, + RepairAction.UPDATE_PROMPT_REVIEW, + False, + True, + "derived-side drift requires requirement-preserving reviewed prompt update", + ) diff --git a/pdd/sync_core/pytest_probe.py b/pdd/sync_core/pytest_probe.py new file mode 100644 index 000000000..0807ec92d --- /dev/null +++ b/pdd/sync_core/pytest_probe.py @@ -0,0 +1,27 @@ +"""Trusted pytest plugin that captures node IDs before repository hooks mutate them.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + + +_OUTPUT_ENV = "PDD_TRUSTED_COLLECTION_OUTPUT" + + +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_collection_modifyitems(items): + """Persist the pre-mutation collection after all inner hooks complete.""" + protected_node_ids = tuple(item.nodeid for item in items) + try: + return (yield) + finally: + output = os.environ.get(_OUTPUT_ENV) + if output: + Path(output).write_text( + json.dumps(protected_node_ids, separators=(",", ":")), + encoding="utf-8", + ) diff --git a/pdd/sync_core/released_checker.py b/pdd/sync_core/released_checker.py new file mode 100644 index 000000000..b152ae108 --- /dev/null +++ b/pdd/sync_core/released_checker.py @@ -0,0 +1,103 @@ +"""Entrypoint for certification from a pinned independently released wheel.""" + +from __future__ import annotations + +import hashlib +import importlib +import os +import sys +import zipfile +from pathlib import Path, PurePosixPath + +from .certificate import CheckerIdentity, checker_identity_from_environment + + +class ReleasedCheckerError(RuntimeError): + """Raised when runtime provenance cannot prove released-wheel execution.""" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_installed_package(wheel: Path, installed_package: Path) -> None: + """Compare every installed PDD member byte-for-byte with the pinned wheel.""" + package_parts = installed_package.parts + package_index = max( + index for index, part in enumerate(package_parts) if part == "pdd" + ) + site_packages = Path(*package_parts[:package_index]) + try: + with zipfile.ZipFile(wheel) as archive: + wheel_members = { + name: archive.read(name) + for name in archive.namelist() + if name.startswith("pdd/") and not name.endswith("/") + } + except (OSError, zipfile.BadZipFile, KeyError) as exc: + raise ReleasedCheckerError("released checker wheel cannot be inspected") from exc + if not wheel_members: + raise ReleasedCheckerError("released checker wheel contains no PDD package") + for member, expected in wheel_members.items(): + installed = site_packages / PurePosixPath(member) + if installed.is_symlink() or not installed.is_file(): + raise ReleasedCheckerError(f"installed checker member is unsafe: {member}") + if installed.read_bytes() != expected: + raise ReleasedCheckerError(f"installed checker member differs from wheel: {member}") + + +def validate_released_checker_runtime( + identity: CheckerIdentity, + wheel_path: Path, + *, + package_path: Path | None = None, + prefix: Path | None = None, + base_prefix: Path | None = None, +) -> None: + """Fail closed unless this process came from the pinned wheel in a venv.""" + wheel = Path(wheel_path) + if wheel.is_symlink() or not wheel.is_file() or wheel.suffix != ".whl": + raise ReleasedCheckerError("released checker wheel path is unsafe") + if _sha256(wheel) != identity.wheel_sha256: + raise ReleasedCheckerError("released checker wheel digest does not match") + + runtime_prefix = Path(prefix or sys.prefix).resolve() + runtime_base = Path(base_prefix or sys.base_prefix).resolve() + installed_package = Path(package_path or __file__).resolve() + if runtime_prefix == runtime_base: + raise ReleasedCheckerError("released checker requires an isolated environment") + try: + installed_package.relative_to(runtime_prefix) + except ValueError as exc: + raise ReleasedCheckerError( + "released checker package is outside the isolated environment" + ) from exc + if "site-packages" not in installed_package.parts: + raise ReleasedCheckerError("released checker was imported from a source checkout") + _verify_installed_package(wheel, installed_package) + + +def main() -> None: + """Verify runtime provenance, then run the strict global certify command.""" + identity = checker_identity_from_environment(require_execution_marker=False) + wheel_value = os.environ.get("PDD_RELEASED_CHECKER_WHEEL_PATH") + if not wheel_value: + raise ReleasedCheckerError("PDD_RELEASED_CHECKER_WHEEL_PATH is required") + validate_released_checker_runtime(identity, Path(wheel_value)) + os.environ["PDD_RELEASED_CHECKER_EXECUTION"] = "1" + + cli = importlib.import_module("pdd.cli").cli + + cli.main( + args=["sync", "certify", *sys.argv[1:]], + prog_name="pdd-sync-checker", + standalone_mode=True, + ) + + +if __name__ == "__main__": + main() diff --git a/pdd/sync_core/reporting.py b/pdd/sync_core/reporting.py new file mode 100644 index 000000000..2a6485893 --- /dev/null +++ b/pdd/sync_core/reporting.py @@ -0,0 +1,358 @@ +"""Strict read-only canonical synchronization reporting.""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from .classifier import classify +from .evidence_store import ( + EvidenceStoreError, + load_attestation, + load_trust_policy, +) +from .fingerprint_store import CorruptFingerprintError, FingerprintStore +from .manifest import ManifestUnit, UnitManifest, build_unit_manifest +from .snapshot import SnapshotError, build_unit_snapshot +from .runner import TRUSTED_RUNNER_VERSION, runner_identity_digest +from .transaction import TransactionError, TransactionManager +from .trust import AttestationError, ValidationEvidence +from .types import ( + BaselineStatus, + InventoryStatus, + SemanticStatus, + SyncVerdict, + VerdictDetails, +) +from .verification import ProfileSet, load_verification_profiles +from .waivers import WaiverSet, load_sync_waivers + + +@dataclass(frozen=True) +class InventoryCounts: + """Complete candidate and managed-denominator counts.""" + + unaccounted_tracked_paths: int + managed_units: int + protected_expected_managed_units: int + managed_waivers: int + invalid: int + + +@dataclass(frozen=True) +class EvidenceCounts: + """Profile and trusted-evidence coverage counts.""" + + trusted_in_sync: int + verification_profile_complete: int + trusted_current_evidence: int + + +@dataclass(frozen=True) +class StatusCounts: + """Independent baseline and semantic failure counts.""" + + drifted: int + unbaselined: int + corrupt: int + unknown: int + conflict: int + failed: int + + +@dataclass(frozen=True) +class CanonicalCounts: + """Machine predicate counts aggregated from canonical unit verdicts.""" + + inventory: InventoryCounts + evidence: EvidenceCounts + statuses: StatusCounts + + def as_flat_dict(self) -> dict[str, int]: + """Return field names used by the external certificate predicate.""" + return { + **self.inventory.__dict__, + **self.evidence.__dict__, + **self.statuses.__dict__, + } + + +@dataclass(frozen=True) +class ReportContext: + """Resolved read-only dependencies shared while classifying units.""" + + root: Path + manifest: UnitManifest + profiles: ProfileSet + store: FingerprintStore + trust_policy: Any + waivers: WaiverSet + now: datetime + + +@dataclass(frozen=True) +class EvidenceExpectation: + """Current closure fields an attestation must match.""" + + unit: ManifestUnit + snapshot_digest: str + profile_digest: str + attestation_ref: str | None + + +@dataclass(frozen=True) +class CanonicalReportOptions: + """Exact refs, scope, trust state, and clock for one report.""" + + base_ref: str = "HEAD" + head_ref: str = "HEAD" + modules: tuple[str, ...] = () + replay_ledger_path: Path | None = None + now: datetime | None = None + + +def _git_sha(root: Path, ref: str) -> str: + result = subprocess.run( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise ValueError(f"cannot resolve Git commit: {ref}") + return result.stdout.strip() + + +def _error_verdict(unit: ManifestUnit, baseline: BaselineStatus, reason: str) -> SyncVerdict: + return SyncVerdict( + unit.unit_id, + InventoryStatus.MANAGED, + baseline, + SemanticStatus.FAILED, + VerdictDetails((), reason), + ) + + +def _evidence( + context: ReportContext, + expectation: EvidenceExpectation, +) -> ValidationEvidence | None: + if not expectation.attestation_ref or context.trust_policy is None: + return None + envelope = load_attestation(context.root, expectation.attestation_ref) + evidence = context.trust_policy.verify( + envelope, envelope.binding, now=context.now + ) + ancestry = subprocess.run( + [ + "git", + "merge-base", + "--is-ancestor", + envelope.binding.checked_sha, + context.manifest.head_ref, + ], + cwd=context.root, + capture_output=True, + check=False, + ) + if ancestry.returncode != 0: + raise AttestationError( + "attestation checked commit is not an ancestor of certified head" + ) + binding = envelope.binding + if ( + binding.subject != expectation.unit.unit_id + or binding.snapshot_digest != expectation.snapshot_digest + or binding.profile_digest != expectation.profile_digest + or binding.base_sha != context.manifest.base_ref + ): + return None + profile = context.profiles.for_unit(expectation.unit.unit_id) + if ( + profile is None + or binding.runner_digest + != runner_identity_digest( + profile, root=context.root, ref=context.manifest.head_ref + ) + or binding.tool_version != TRUSTED_RUNNER_VERSION + ): + raise AttestationError("attestation runner identity is not protected") + return evidence + + +def _unit_verdict(context: ReportContext, unit: ManifestUnit) -> SyncVerdict: + profile = context.profiles.for_unit(unit.unit_id) + if profile is None: + return _error_verdict(unit, BaselineStatus.CORRUPT, "profile is missing") + try: + snapshot = build_unit_snapshot(context.root, context.manifest, unit, profile) + baseline = context.store.load(unit.unit_id) + attestation_ref = baseline.attestation_ref if baseline else None + evidence = _evidence( + context, + EvidenceExpectation( + unit, + snapshot.digest(), + profile.profile_digest, + attestation_ref, + ), + ) + return classify(snapshot, baseline, profile, evidence) + except CorruptFingerprintError as exc: + return _error_verdict(unit, BaselineStatus.CORRUPT, str(exc)) + except (SnapshotError, EvidenceStoreError, AttestationError) as exc: + return _error_verdict(unit, BaselineStatus.DRIFTED, str(exc)) + + +def _counts( + manifest: UnitManifest, + profiles: ProfileSet, + waivers: WaiverSet, + verdicts: tuple[SyncVerdict, ...], + errors: Iterable[str], +) -> CanonicalCounts: + errors_present = bool(tuple(errors)) + inventory = InventoryCounts( + len(manifest.unaccounted_tracked_paths), + len(manifest.managed_units), + len(manifest.expected_managed), + waivers.managed_count, + sum(item.inventory is InventoryStatus.INVALID for item in manifest.candidates) + + int(errors_present), + ) + evidence = EvidenceCounts( + sum(verdict.in_sync for verdict in verdicts), + sum(profile.complete for profile in profiles.profiles), + sum(verdict.evidence_complete for verdict in verdicts), + ) + statuses = StatusCounts( + sum(verdict.baseline is BaselineStatus.DRIFTED for verdict in verdicts), + sum(verdict.baseline is BaselineStatus.UNBASELINED for verdict in verdicts), + sum(verdict.baseline is BaselineStatus.CORRUPT for verdict in verdicts), + sum(verdict.semantic is SemanticStatus.UNKNOWN for verdict in verdicts), + sum(verdict.semantic is SemanticStatus.CONFLICT for verdict in verdicts), + sum(verdict.semantic is SemanticStatus.FAILED for verdict in verdicts), + ) + return CanonicalCounts(inventory, evidence, statuses) + + +def _predicate(counts: CanonicalCounts) -> bool: + inventory = counts.inventory + evidence = counts.evidence + statuses = counts.statuses + return ( + inventory.unaccounted_tracked_paths == 0 + and inventory.managed_units > 0 + and inventory.managed_units == inventory.protected_expected_managed_units + and inventory.managed_waivers == 0 + and evidence.trusted_in_sync == inventory.managed_units + and evidence.verification_profile_complete == inventory.managed_units + and evidence.trusted_current_evidence == inventory.managed_units + and statuses.drifted == 0 + and statuses.unbaselined == 0 + and statuses.corrupt == 0 + and statuses.unknown == 0 + and statuses.conflict == 0 + and statuses.failed == 0 + and inventory.invalid == 0 + ) + + +def _report_context( + root: Path, + options: CanonicalReportOptions, +) -> tuple[ReportContext, list[str], tuple[str, ...]]: + """Resolve protected inputs, trust policy, store, and recovery state.""" + base_sha = _git_sha(root, options.base_ref) + head_sha = _git_sha(root, options.head_ref) + manifest = build_unit_manifest(root, base_ref=base_sha, head_ref=head_sha) + profiles = load_verification_profiles(root, manifest) + now = options.now or datetime.now(timezone.utc) + waivers = load_sync_waivers(root, manifest, now=now) + errors = list( + manifest.invalid_reasons + + profiles.invalid_reasons + + waivers.invalid_reasons + ) + ledger = options.replay_ledger_path + if ledger is None: + configured = os.environ.get("PDD_ATTESTATION_REPLAY_LEDGER") + ledger = Path(configured) if configured else None + trust_policy = None + if ledger is None: + errors.append("external attestation replay ledger is not configured") + else: + try: + trust_policy = load_trust_policy( + root, base_sha, replay_ledger_path=ledger + ).verifier + except EvidenceStoreError as exc: + errors.append(str(exc)) + try: + recovery = TransactionManager(root).incomplete() + except TransactionError as exc: + recovery = () + errors.append(str(exc)) + if recovery: + errors.append("RECOVERY_REQUIRED: " + ", ".join(recovery)) + context = ReportContext( + root, + manifest, + profiles, + FingerprintStore(root), + trust_policy, + waivers, + now, + ) + return context, errors, recovery + + +def build_canonical_report( + root: Path, + options: CanonicalReportOptions = CanonicalReportOptions(), +) -> dict[str, Any]: + """Build a strict trusted report; never mutate repository-managed state.""" + repository_root = Path(root).resolve() + context, errors, recovery = _report_context(repository_root, options) + manifest = context.manifest + profiles = context.profiles + wanted = set(options.modules) + selected = tuple( + unit + for unit in manifest.managed_units + if not wanted + or unit.unit_id.prompt_relpath.stem.rsplit("_", 1)[0] in wanted + or unit.unit_id.prompt_relpath.as_posix() in wanted + ) + verdicts = tuple(_unit_verdict(context, unit) for unit in selected) + counts = _counts(manifest, profiles, context.waivers, verdicts, errors) + return { + "schema_version": 1, + "ok": _predicate(counts) and len(selected) == len(manifest.managed_units), + "project_root": str(repository_root), + "repository_id": manifest.repository_id, + "base_sha": manifest.base_ref, + "head_sha": manifest.head_ref, + "manifest_digest": manifest.digest(), + "counts": counts.as_flat_dict(), + "errors": sorted(set(errors)), + "recovery_required": list(recovery), + "units": [ + { + "subject": verdict.subject.prompt_relpath.as_posix(), + "inventory": verdict.inventory.value, + "baseline": verdict.baseline.value, + "semantic": verdict.semantic.value, + "in_sync": verdict.in_sync, + "evidence_complete": verdict.evidence_complete, + "changed_roles": list(verdict.changed_roles), + "reason": verdict.reason, + } + for verdict in verdicts + ], + } diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py new file mode 100644 index 000000000..e0e222093 --- /dev/null +++ b/pdd/sync_core/runner.py @@ -0,0 +1,688 @@ +"""Trusted validator adapters and pass-only normalized evidence outcomes.""" + +from __future__ import annotations + +import hashlib +import ast +import json +import os +import re +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path, PurePosixPath + +from .trust import ( + AttestationBinding, + AttestationEnvelope, + AttestationRequest, + AttestationSigner, +) +from .types import ( + EvidenceOutcome, + ObligationEvidence, + UnitId, + VerificationObligation, + VerificationProfile, +) + + +TRUSTED_RUNNER_VERSION = "pdd-trusted-runner-v2" +PYTEST_CONFIG_PATHS = ( + PurePosixPath("pytest.ini"), + PurePosixPath("pyproject.toml"), + PurePosixPath("tox.ini"), + PurePosixPath("setup.cfg"), +) +PYTEST_PROTECTED_FLAGS = ("--strict-config", "--strict-markers", "-ra") + + +@dataclass(frozen=True) +class RunnerConfig: + """Protected execution limits for trusted validation adapters.""" + + timeout_seconds: int = 300 + + +@dataclass(frozen=True) +class RunnerExecution: + """Normalized outcome and command identity for one obligation.""" + + obligation_id: str + outcome: EvidenceOutcome + command_digest: str + detail: str + + +@dataclass(frozen=True) +class RunBinding: + """Snapshot and Git closure supplied to all profile obligations.""" + + snapshot_digest: str + base_sha: str + head_sha: str + + +@dataclass(frozen=True) +class AttestationIssue: + """Trusted signer and unique issuance fields for one profile run.""" + + signer: AttestationSigner + attestation_id: str + nonce: str + issued_at: datetime + + +def _git_blob(root: Path, ref: str, path: PurePosixPath) -> bytes | None: + result = subprocess.run( + ["git", "show", f"{ref}:{path.as_posix()}"], + cwd=root, + capture_output=True, + check=False, + ) + return result.stdout if result.returncode == 0 else None + + +def _local_module_paths( + root: Path, ref: str, source_path: PurePosixPath, source: bytes +) -> set[PurePosixPath]: + # pylint: disable=too-many-locals + """Resolve repository-local Python imports without executing candidate code.""" + try: + tree = ast.parse(source) + except (SyntaxError, UnicodeDecodeError): + return set() + modules: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + prefix = "." * node.level + modules.add(prefix + node.module) + elif isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "pytest_plugins" + for target in node.targets + ): + values = node.value.elts if isinstance(node.value, (ast.List, ast.Tuple)) else () + modules.update( + item.value for item in values if isinstance(item, ast.Constant) + and isinstance(item.value, str) + ) + resolved: set[PurePosixPath] = set() + for module in modules: + level = len(module) - len(module.lstrip(".")) + name = module.lstrip(".") + base = source_path.parent + for _ in range(max(level - 1, 0)): + base = base.parent + module_path = PurePosixPath(*name.split(".")) if name else PurePosixPath() + candidates = ( + base / module_path.with_suffix(".py"), + base / module_path / "__init__.py", + module_path.with_suffix(".py"), + module_path / "__init__.py", + ) + resolved.update(path for path in candidates if _git_blob(root, ref, path) is not None) + return resolved + + +def _pytest_support_closure( + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] +) -> tuple[tuple[PurePosixPath, bytes], ...]: + """Return config, conftest, and transitive local-import blobs for pytest.""" + pending = list(test_paths) + paths = {path for path in PYTEST_CONFIG_PATHS if _git_blob(root, ref, path) is not None} + for test_path in test_paths: + parent = test_path.parent + while parent != PurePosixPath("."): + candidate = parent / "conftest.py" + if _git_blob(root, ref, candidate) is not None: + paths.add(candidate) + parent = parent.parent + root_conftest = PurePosixPath("conftest.py") + if _git_blob(root, ref, root_conftest) is not None: + paths.add(root_conftest) + pending.extend(paths) + visited: set[PurePosixPath] = set() + while pending: + path = pending.pop() + if path in visited: + continue + visited.add(path) + source = _git_blob(root, ref, path) + if source is None or path.suffix != ".py": + continue + discovered = _local_module_paths(root, ref, path, source) - visited + paths.update(discovered) + pending.extend(discovered) + blobs = ((path, _git_blob(root, ref, path)) for path in sorted(paths)) + return tuple((path, blob) for path, blob in blobs if blob is not None) + + +def _support_digest( + root: Path, ref: str, profile: VerificationProfile +) -> tuple[str, tuple[PurePosixPath, ...]]: + tests = tuple( + path + for obligation in profile.obligations + if obligation.validator_id == "pytest" + for path in obligation.artifact_paths + ) + closure = _pytest_support_closure(root, ref, tests) + digest = hashlib.sha256() + for path, content in closure: + digest.update(path.as_posix().encode() + b"\0" + content + b"\0") + return digest.hexdigest(), tuple(path for path, _content in closure) + + +def _config_loads_plugin(root: Path, ref: str) -> bool: + """Reject repository-configured plugins until profiles bind plugin identities.""" + pattern = re.compile(r"(?:^|[\s\"'])-p(?:[=\s]+)[A-Za-z0-9_.-]+") + for path in PYTEST_CONFIG_PATHS: + content = _git_blob(root, ref, path) + if content is None: + continue + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + return True + if pattern.search(text): + return True + return False + + +def runner_identity_digest( + profile: VerificationProfile, *, root: Path | None = None, ref: str = "HEAD" +) -> str: + """Bind evidence to protected adapters, configs, and exact artifact scopes.""" + payload = { + "tool_version": TRUSTED_RUNNER_VERSION, + "pytest_command": [ + sys.executable, + "-m", + "pytest", + "-q", + *PYTEST_PROTECTED_FLAGS, + "", + "--junitxml=", + ], + "pytest_collection_command": [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "--strict-config", + "--strict-markers", + "-p", + "pdd.sync_core.pytest_probe", + "", + ], + "pytest_environment": {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, + "obligations": [ + { + "id": item.obligation_id, + "kind": item.kind, + "validator": item.validator_id, + "config": item.validator_config_digest, + "paths": [path.as_posix() for path in item.artifact_paths], + } + for item in profile.obligations + ], + } + if root is not None: + payload["support_closure_digest"] = _support_digest(root, ref, profile)[0] + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def _changed_paths( + root: Path, + base_sha: str, + head_sha: str, + paths: tuple[PurePosixPath, ...], +) -> set[str]: + if not paths: + return set() + changed = set() + revisions = [(base_sha, head_sha), (head_sha,)] + for revision in revisions: + result = subprocess.run( + [ + "git", + "diff", + "--name-only", + *revision, + "--", + *(path.as_posix() for path in paths), + ], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return {path.as_posix() for path in paths} + changed.update(line for line in result.stdout.splitlines() if line) + return changed + + +def _junit_outcome( + path: Path, returncode: int, output: str, minimum_tests: int +) -> tuple[EvidenceOutcome, str]: + try: + root = ET.parse(path).getroot() + except (ET.ParseError, OSError) as exc: + return EvidenceOutcome.COLLECTION_ERROR, f"cannot parse JUnit result: {exc}" + suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) + totals = { + key: sum(int(suite.attrib.get(key, "0")) for suite in suites) + for key in ("tests", "failures", "errors", "skipped") + } + if totals["tests"] == 0: + outcome, detail = EvidenceOutcome.NOT_COLLECTED, "zero tests collected" + elif totals["tests"] < minimum_tests: + outcome, detail = EvidenceOutcome.NOT_COLLECTED, ( + f"executed {totals['tests']} of at least {minimum_tests} declared tests" + ) + elif " deselected" in output: + outcome, detail = ( + EvidenceOutcome.NOT_COLLECTED, + "pytest deselected declared tests", + ) + elif "XPASS" in output: + outcome, detail = EvidenceOutcome.XFAIL, "pytest reported an unexpected pass" + elif totals["errors"]: + outcome, detail = EvidenceOutcome.ERROR, f"{totals['errors']} test errors" + elif totals["failures"] or returncode: + outcome, detail = EvidenceOutcome.FAIL, f"{totals['failures']} test failures" + elif totals["skipped"]: + outcome, detail = ( + EvidenceOutcome.SKIP, + f"{totals['skipped']} tests skipped or xfailed", + ) + else: + outcome, detail = EvidenceOutcome.PASS, f"{totals['tests']} tests passed" + return outcome, detail + + +def _pytest_environment() -> dict[str, str]: + """Return the protected credential-free pytest process environment.""" + return { + key: value + for key, value in os.environ.items() + if not any( + marker in key.upper() + for marker in ( + "CREDENTIAL", + "PASSWORD", + "SECRET", + "SIGNING_KEY", + "TOKEN", + ) + ) + and key not in {"PYTEST_ADDOPTS", "PYTHONPATH"} + } | {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PYTHONNOUSERSITE": "1"} + + +def _run_test_node( + root: Path, + node_id: str, + timeout_seconds: int, +) -> RunnerExecution: + command = [ + sys.executable, + "-m", + "pytest", + "-q", + *PYTEST_PROTECTED_FLAGS, + node_id, + ] + command_digest = hashlib.sha256( + json.dumps(command, separators=(",", ":")).encode() + ).hexdigest() + with tempfile.TemporaryDirectory(prefix="pdd-trusted-runner-") as directory: + junit = Path(directory) / "junit.xml" + try: + result = subprocess.run( + [*command, f"--junitxml={junit}"], + cwd=root, + capture_output=True, + text=True, + check=False, + timeout=timeout_seconds, + env=_pytest_environment(), + ) + except subprocess.TimeoutExpired: + return RunnerExecution( + node_id, + EvidenceOutcome.TIMEOUT, + command_digest, + "test execution timed out", + ) + outcome, detail = _junit_outcome( + junit, + result.returncode, + result.stdout + "\n" + result.stderr, + 1, + ) + return RunnerExecution(node_id, outcome, command_digest, detail) + + +def _collect_node_ids( + root: Path, + path: PurePosixPath, + timeout_seconds: int, +) -> tuple[RunnerExecution, tuple[str, ...]]: + """Collect exact pytest node IDs through the protected adapter.""" + with tempfile.TemporaryDirectory(prefix="pdd-trusted-collection-") as directory: + collection_output = Path(directory) / "node-ids.json" + command = [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "--strict-config", + "--strict-markers", + "-p", + "pdd.sync_core.pytest_probe", + path.as_posix(), + ] + digest = hashlib.sha256( + json.dumps(command, separators=(",", ":")).encode() + ).hexdigest() + try: + result = subprocess.run( + command, + cwd=root, + capture_output=True, + text=True, + check=False, + timeout=timeout_seconds, + env=_pytest_environment() + | {"PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output)}, + ) + except subprocess.TimeoutExpired: + return ( + RunnerExecution( + path.as_posix(), + EvidenceOutcome.TIMEOUT, + digest, + "test collection timed out", + ), + (), + ) + try: + payload = json.loads(collection_output.read_text(encoding="utf-8")) + if not isinstance(payload, list) or not all( + isinstance(item, str) for item in payload + ): + raise ValueError("node ID payload is malformed") + node_ids = tuple(sorted(payload)) + except (OSError, ValueError, json.JSONDecodeError): + return ( + RunnerExecution( + path.as_posix(), + EvidenceOutcome.COLLECTION_ERROR, + digest, + "trusted collection probe produced no valid node IDs", + ), + (), + ) + if not node_ids and result.returncode in {0, 5}: + outcome = EvidenceOutcome.NOT_COLLECTED + detail = "zero protected node IDs collected" + elif result.returncode != 0: + outcome = EvidenceOutcome.COLLECTION_ERROR + detail = "pytest collection failed" + else: + outcome = EvidenceOutcome.PASS + detail = f"{len(node_ids)} protected node IDs collected" + return RunnerExecution(path.as_posix(), outcome, digest, detail), node_ids + + +def _collect_at_base( + root: Path, + base_sha: str, + paths: tuple[PurePosixPath, ...], + timeout_seconds: int, +) -> tuple[tuple[RunnerExecution, ...], tuple[str, ...]]: + """Collect node IDs from a fresh exact protected-base clone.""" + with tempfile.TemporaryDirectory(prefix="pdd-runner-protected-base-") as directory: + clone = Path(directory) / "repository" + clone_result = subprocess.run( + ["git", "clone", "-q", "--no-local", "--no-checkout", str(root), str(clone)], + capture_output=True, + text=True, + check=False, + ) + if clone_result.returncode != 0: + error = RunnerExecution( + "protected-base-collection", + EvidenceOutcome.COLLECTION_ERROR, + hashlib.sha256(base_sha.encode()).hexdigest(), + "cannot create protected-base runner clone", + ) + return (error,), () + checkout = subprocess.run( + ["git", "checkout", "-q", "--detach", base_sha], + cwd=clone, + capture_output=True, + check=False, + ) + if checkout.returncode != 0: + error = RunnerExecution( + "protected-base-collection", + EvidenceOutcome.COLLECTION_ERROR, + hashlib.sha256(base_sha.encode()).hexdigest(), + "cannot checkout protected base for collection", + ) + return (error,), () + collected = tuple( + _collect_node_ids(clone, path, timeout_seconds) for path in paths + ) + return ( + tuple(item[0] for item in collected), + tuple(node_id for item in collected for node_id in item[1]), + ) + + +def _dirty_pytest_support(root: Path) -> set[str]: + """Return live paths capable of changing pytest outside the Git closure.""" + result = subprocess.run( + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], + cwd=root, + capture_output=True, + check=False, + ) + if result.returncode != 0: + return {""} + dirty: set[str] = set() + for field in result.stdout.decode(errors="surrogateescape").split("\0"): + path = field[3:] if len(field) >= 4 else "" + relpath = PurePosixPath(path) + if ( + relpath.name == "conftest.py" + or relpath in PYTEST_CONFIG_PATHS + or (relpath.suffix == ".py" and "tests" in relpath.parts) + ): + dirty.add(path) + return dirty + + +def _combine( + obligation: VerificationObligation, + executions: tuple[RunnerExecution, ...], +) -> RunnerExecution: + order = ( + EvidenceOutcome.TIMEOUT, + EvidenceOutcome.COLLECTION_ERROR, + EvidenceOutcome.ERROR, + EvidenceOutcome.FAIL, + EvidenceOutcome.SKIP, + EvidenceOutcome.XFAIL, + EvidenceOutcome.NOT_COLLECTED, + EvidenceOutcome.PASS, + ) + outcomes = {execution.outcome for execution in executions} + outcome = next(item for item in order if item in outcomes) + digest = hashlib.sha256( + "".join(item.command_digest for item in executions).encode() + ).hexdigest() + detail = "; ".join(item.detail for item in executions) + return RunnerExecution(obligation.obligation_id, outcome, digest, detail) + + +def _obligation_preflight( + root: Path, + obligation: VerificationObligation, + base_sha: str, + head_sha: str, +) -> RunnerExecution | None: + """Return a normalized fail-closed result before executing pytest.""" + if obligation.kind.casefold() != "test" or obligation.validator_id != "pytest": + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "no trusted runner adapter is registered", + ) + if not obligation.artifact_paths: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.NOT_COLLECTED, + obligation.validator_config_digest, + "test obligation declares no artifact paths", + ) + if _config_loads_plugin(root, base_sha) or _config_loads_plugin(root, head_sha): + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "repository-configured pytest plugins are not bound by this adapter", + ) + return None + + +def _protected_node_ids( + root: Path, + obligation: VerificationObligation, + base_sha: str, + timeout_seconds: int, +) -> tuple[ + RunnerExecution | None, + tuple[RunnerExecution, ...], + tuple[str, ...], +]: + """Compare protected-base and checked-head pre-hook collection identities.""" + base_executions, base_node_ids = _collect_at_base( + root, base_sha, obligation.artifact_paths, timeout_seconds + ) + head_collected = tuple( + _collect_node_ids(root, path, timeout_seconds) + for path in obligation.artifact_paths + ) + head_executions = tuple(item[0] for item in head_collected) + head_node_ids = tuple(node_id for item in head_collected for node_id in item[1]) + collection_executions = base_executions + head_executions + if any(item.outcome is not EvidenceOutcome.PASS for item in collection_executions): + return _combine(obligation, collection_executions), (), () + if base_node_ids != head_node_ids: + mismatch = RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.QUARANTINED, + hashlib.sha256("".join(base_node_ids + head_node_ids).encode()).hexdigest(), + "checked-head pytest node IDs differ from protected base", + ) + return mismatch, (), () + return None, collection_executions, head_node_ids + + +def run_obligation( + root: Path, + obligation: VerificationObligation, + *, + base_sha: str, + head_sha: str, + config: RunnerConfig, +) -> RunnerExecution: + """Run one protected obligation with changed-test self-certification guards.""" + preflight = _obligation_preflight(root, obligation, base_sha, head_sha) + if preflight is not None: + return preflight + profile = VerificationProfile( + UnitId("runner-closure", PurePosixPath("closure.prompt"), "python"), + (obligation,), + obligation.requirement_ids, + "closure", + ) + _digest, support_paths = _support_digest(root, head_sha, profile) + protected_paths = tuple(sorted(set(obligation.artifact_paths) | set(support_paths))) + changed = _changed_paths(root, base_sha, head_sha, protected_paths) + changed.update(_dirty_pytest_support(root)) + if changed: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.QUARANTINED, + obligation.validator_config_digest, + "candidate-modified test cannot solely certify itself: " + + ", ".join(sorted(changed)), + ) + collection_error, collection_executions, head_node_ids = _protected_node_ids( + root, obligation, base_sha, config.timeout_seconds + ) + if collection_error is not None: + return collection_error + executions = tuple( + _run_test_node(root, node_id, config.timeout_seconds) + for node_id in head_node_ids + ) + return _combine(obligation, collection_executions + executions) + + +def run_profile( + root: Path, + profile: VerificationProfile, + binding: RunBinding, + issuance: AttestationIssue, + config: RunnerConfig = RunnerConfig(), +) -> tuple[AttestationEnvelope, tuple[RunnerExecution, ...]]: + """Execute every obligation and issue one complete signed attestation.""" + executions = tuple( + run_obligation( + root, + obligation, + base_sha=binding.base_sha, + head_sha=binding.head_sha, + config=config, + ) + for obligation in profile.obligations + ) + runner_digest = runner_identity_digest(profile, root=root, ref=binding.head_sha) + binding = AttestationBinding( + profile.unit_id, + binding.snapshot_digest, + profile.profile_digest, + runner_digest, + TRUSTED_RUNNER_VERSION, + binding.base_sha, + binding.head_sha, + ) + results = tuple( + ObligationEvidence(item.obligation_id, item.outcome) for item in executions + ) + envelope = issuance.signer.issue( + AttestationRequest( + issuance.attestation_id, + binding, + results, + issuance.nonce, + issuance.issued_at, + ) + ) + return envelope, executions diff --git a/pdd/sync_core/scenario_contract.py b/pdd/sync_core/scenario_contract.py new file mode 100644 index 000000000..b52e692b7 --- /dev/null +++ b/pdd/sync_core/scenario_contract.py @@ -0,0 +1,17 @@ +"""Stable required scenario identities shared by checker and launcher.""" + +REQUIRED_SCENARIOS = frozenset( + { + "source-edit-matrix", + "missing-corrupt-delete-mode", + "transaction-crash-race-recovery", + "forged-stale-replayed-revoked-evidence", + "complete-base-head-inventory", + "trusted-runner-outcomes", + "transactional-canonical-report", + "merge-group-base-movement-and-stale-repair", + "built-wheel-clean-environment", + "pdd-cloud-real-consumer-canary", + "released-checker-owned-scenario-harness", + } +) diff --git a/pdd/sync_core/scenario_harness.py b/pdd/sync_core/scenario_harness.py new file mode 100644 index 000000000..0f6cb5c85 --- /dev/null +++ b/pdd/sync_core/scenario_harness.py @@ -0,0 +1,650 @@ +"""Checker-owned lifecycle scenarios shipped inside the released wheel.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path, PurePosixPath +from typing import Callable + +from .certificate import count_vendored_sync_semantics +from .classifier import classify +from .identity import initialize_repository_identity +from .manifest import build_unit_manifest +from .reporting import CanonicalReportOptions, build_canonical_report +from .runner import AttestationIssue, RunBinding, RunnerConfig, run_profile +from .scenario_contract import REQUIRED_SCENARIOS +from .transaction import PlannedWrite, TransactionConflict, TransactionManager +from .trust import ( + AttestationBinding, + AttestationError, + AttestationRequest, + AttestationSigner, + AttestationTrustPolicy, +) +from .types import ( + ArtifactSnapshot, + BaselineStatus, + EvidenceOutcome, + FingerprintProvenance, + FingerprintRecord, + SemanticStatus, + UnitId, + UnitSnapshot, + VerificationObligation, + VerificationProfile, + ObligationEvidence, +) + + +@dataclass(frozen=True) +# pylint: disable=too-many-instance-attributes +class ScenarioResult: + """One checker-owned scenario outcome.""" + + scenario_id: str + status: str + detail: str = "" + required_tests_skipped_or_xfailed: int = 0 + collection_errors: int = 0 + timeouts: int = 0 + post_repair_second_run_writes: int = 0 + post_merge_tree_changes: int = 0 + + +def _profile(unit: UnitId) -> VerificationProfile: + obligation = VerificationObligation( + "tests", + "test", + "pytest", + "checker-owned-config", + ("REQ-1",), + (PurePosixPath("tests/test_widget.py"),), + ) + return VerificationProfile(unit, (obligation,), ("REQ-1",), "profile-v1") + + +def _snapshot(unit: UnitId, **changes: str | None) -> UnitSnapshot: + values: dict[str, str | None] = { + "prompt": "prompt-v1", + "include": "include-v1", + "code": "code-v1", + "example": "example-v1", + "test": "test-v1", + } + values.update(changes) + paths = { + "prompt": "prompts/widget_python.prompt", + "include": "docs/widget.md", + "code": "src/widget.py", + "example": "examples/widget.py", + "test": "tests/test_widget.py", + } + return UnitSnapshot( + unit, + tuple( + ArtifactSnapshot( + role, + PurePosixPath(paths[role]), + digest, + "100644" if digest is not None else None, + ) + for role, digest in values.items() + ), + "manifest-v1", + "dependency-v1", + "profile-v1", + ) + + +def _baseline(snapshot: UnitSnapshot) -> FingerprintRecord: + provenance = FingerprintProvenance( + "generated", + "pdd sync widget", + "checker-transaction", + "checked-head", + "2026-07-10T00:00:00+00:00", + "released-checker", + ) + return FingerprintRecord( + snapshot, 2, 2, provenance, SemanticStatus.VERIFIED, "checker-attestation" + ) + + +def _git(root: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", *arguments], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stderr.strip() or "Git fixture command failed") + return completed.stdout.strip() + + +def _git_fixture(root: Path) -> None: + _git(root, "init", "-q") + _git(root, "config", "user.email", "checker@example.com") + _git(root, "config", "user.name", "Released Checker") + + +def _commit(root: Path, message: str) -> str: + _git(root, "add", ".") + _git(root, "commit", "-qm", message) + return _git(root, "rev-parse", "HEAD") + + +def _source_edit_matrix(_arguments: argparse.Namespace) -> ScenarioResult: + unit = UnitId("checker-fixture", PurePosixPath("prompts/widget_python.prompt"), "python") + baseline = _snapshot(unit) + profile = _profile(unit) + for role in ("prompt", "include", "code", "example", "test"): + verdict = classify( + _snapshot(unit, **{role: f"{role}-v2"}), + _baseline(baseline), + profile, + None, + ) + if verdict.in_sync or role not in verdict.changed_roles: + raise AssertionError(f"{role}-only edit was not blocking") + simultaneous = classify( + _snapshot(unit, prompt="prompt-v2", code="code-v2"), + _baseline(baseline), + profile, + None, + ) + if simultaneous.semantic is not SemanticStatus.CONFLICT: + raise AssertionError("simultaneous prompt/code edit was not a conflict") + return ScenarioResult("source-edit-matrix", "PASS") + + +def _missing_corrupt_delete_mode(_arguments: argparse.Namespace) -> ScenarioResult: + unit = UnitId("checker-fixture", PurePosixPath("prompts/widget_python.prompt"), "python") + baseline = _snapshot(unit) + profile = _profile(unit) + missing = classify( + _snapshot(unit, code=None), _baseline(baseline), profile, None + ) + if missing.semantic is not SemanticStatus.FAILED: + raise AssertionError("missing required artifact did not fail") + corrupt_artifact = classify( + _snapshot(unit, code="corrupt-content"), + _baseline(baseline), + profile, + None, + ) + if corrupt_artifact.baseline is not BaselineStatus.DRIFTED: + raise AssertionError("corrupt artifact content was not detected") + invalid_fingerprint = FingerprintRecord( + baseline, + 1, + 2, + _baseline(baseline).provenance, + SemanticStatus.VERIFIED, + "checker-attestation", + ) + corrupt_fingerprint = classify(baseline, invalid_fingerprint, profile, None) + if corrupt_fingerprint.baseline is not BaselineStatus.CORRUPT: + raise AssertionError("corrupt fingerprint was not rejected") + + def relocated(path: str, *, manifest: str = "manifest-v1") -> UnitSnapshot: + artifacts = tuple( + ArtifactSnapshot( + item.role, + PurePosixPath(path) if item.role == "code" else item.relpath, + item.digest, + item.git_mode, + item.required, + ) + for item in baseline.artifacts + ) + return UnitSnapshot( + unit, + artifacts, + manifest, + baseline.dependency_snapshot_digest, + baseline.verification_profile_digest, + ) + + renamed = classify( + relocated("src/widget_renamed.py"), _baseline(baseline), profile, None + ) + if ( + renamed.baseline is not BaselineStatus.DRIFTED + or "code" not in renamed.changed_roles + ): + raise AssertionError("renamed artifact was not detected") + retargeted = classify( + relocated("generated/widget.py", manifest="manifest-retargeted"), + _baseline(baseline), + profile, + None, + ) + if not {"code", "manifest"}.issubset(retargeted.changed_roles): + raise AssertionError("retargeted artifact declaration was not detected") + changed_mode = tuple( + ArtifactSnapshot( + item.role, + item.relpath, + item.digest, + "100755" if item.role == "code" else item.git_mode, + ) + for item in baseline.artifacts + ) + mode_snapshot = UnitSnapshot( + unit, + changed_mode, + baseline.manifest_digest, + baseline.dependency_snapshot_digest, + baseline.verification_profile_digest, + ) + mode = classify(mode_snapshot, _baseline(baseline), profile, None) + if "code" not in mode.changed_roles: + raise AssertionError("mode-only change was not detected") + return ScenarioResult("missing-corrupt-delete-mode", "PASS") + + +def _prepared_recovery(writes: tuple[PlannedWrite, ...]) -> None: + with tempfile.TemporaryDirectory(prefix="pdd-checker-prepared-") as directory: + root = Path(directory) + (root / "src").mkdir() + target = root / "src/widget.py" + target.write_text("old\n") + manager = TransactionManager(root) + manager.prepare("prepared", writes) + recovered = manager.recover("prepared") + if recovered.phase.value != "ROLLED_BACK" or target.read_text() != "old\n": + raise AssertionError("PREPARED journal recovery changed a destination") + + +def _recover_after_crashes(writes: tuple[PlannedWrite, ...]) -> None: + crash_events = ( + "after_committing", + "after_install:0", + "after_install:1", + "after_commit", + ) + for index, event in enumerate(crash_events): + with tempfile.TemporaryDirectory(prefix="pdd-checker-transaction-") as directory: + root = Path(directory) + (root / "src").mkdir() + (root / "src/widget.py").write_text("old\n") + manager = TransactionManager(root) + manager.prepare(f"crash-{index}", writes) + + def crash(observed: str, expected: str = event) -> None: + if observed == expected: + raise SystemExit("injected crash") + + try: + manager.commit(f"crash-{index}", crash_hook=crash) + except SystemExit: + pass + manager.recover(f"crash-{index}") + if (root / "src/widget.py").read_text() != "new\n": + raise AssertionError(f"recovery failed after {event}") + if (root / ".pdd/evidence/widget.json").read_text() != "{}\n": + raise AssertionError(f"recovery left a partial state after {event}") + + +def _external_write_race(writes: tuple[PlannedWrite, ...]) -> None: + with tempfile.TemporaryDirectory(prefix="pdd-checker-race-") as directory: + root = Path(directory) + (root / "src").mkdir() + target = root / "src/widget.py" + target.write_text("old\n") + manager = TransactionManager(root) + manager.prepare("external-race", writes) + target.write_text("external\n") + try: + manager.commit("external-race") + except TransactionConflict: + pass + else: + raise AssertionError("external write race was not rejected") + + +def _concurrent_sync_race(writes: tuple[PlannedWrite, ...]) -> None: + with tempfile.TemporaryDirectory(prefix="pdd-checker-concurrent-") as directory: + root = Path(directory) + (root / "src").mkdir() + (root / "src/widget.py").write_text("old\n") + first = TransactionManager(root) + second = TransactionManager(root) + first.prepare("concurrent-a", writes) + second.prepare("concurrent-b", writes) + barrier = threading.Barrier(2) + + def commit(manager: TransactionManager, transaction_id: str) -> str: + barrier.wait(timeout=10) + try: + manager.commit(transaction_id) + except TransactionConflict: + return "CONFLICT" + return "COMMITTED" + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = ( + executor.submit(commit, first, "concurrent-a"), + executor.submit(commit, second, "concurrent-b"), + ) + outcomes = tuple(sorted(future.result() for future in futures)) + if outcomes != ("COMMITTED", "CONFLICT"): + raise AssertionError(f"concurrent sync was not serialized: {outcomes}") + + +def _transaction_crash_race(_arguments: argparse.Namespace) -> ScenarioResult: + writes = ( + PlannedWrite(PurePosixPath("src/widget.py"), b"new\n", "100644"), + PlannedWrite(PurePosixPath(".pdd/evidence/widget.json"), b"{}\n", "100644"), + ) + _prepared_recovery(writes) + _recover_after_crashes(writes) + _external_write_race(writes) + _concurrent_sync_race(writes) + return ScenarioResult("transaction-crash-race-recovery", "PASS") + + +def _evidence_guards(_arguments: argparse.Namespace) -> ScenarioResult: + now = datetime.now(timezone.utc) + signer = AttestationSigner("checker-fixture", b"e" * 32) + unit = UnitId("checker-fixture", PurePosixPath("prompts/widget_python.prompt"), "python") + binding = AttestationBinding( + unit, "snapshot", "profile", "runner", "checker", "base", "head" + ) + envelope = signer.issue( + AttestationRequest( + "attestation", + binding, + (ObligationEvidence("tests", EvidenceOutcome.PASS),), + "nonce", + now, + ) + ) + policy = AttestationTrustPolicy({"checker-fixture": signer.public_key_bytes()}) + policy.verify(envelope, binding, now=now) + for candidate, expected in ( + (envelope, binding), + ( + signer.issue( + AttestationRequest( + "expired", + binding, + envelope.results, + "expired-nonce", + now - timedelta(hours=2), + timedelta(minutes=1), + ) + ), + binding, + ), + ): + try: + policy.verify(candidate, expected, now=now) + except AttestationError: + continue + raise AssertionError("replayed or stale evidence was accepted") + revoked = AttestationTrustPolicy( + {"checker-fixture": signer.public_key_bytes()}, + revoked_issuers=frozenset({"checker-fixture"}), + ) + try: + revoked.verify(envelope, binding, now=now) + except AttestationError: + pass + else: + raise AssertionError("revoked evidence issuer was accepted") + return ScenarioResult("forged-stale-replayed-revoked-evidence", "PASS") + + +def _complete_inventory(_arguments: argparse.Namespace) -> ScenarioResult: + with tempfile.TemporaryDirectory(prefix="pdd-checker-inventory-") as directory: + root = Path(directory) + _git_fixture(root) + initialize_repository_identity( + root, "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + ) + (root / "prompts").mkdir() + (root / "src").mkdir() + (root / "prompts/widget_python.prompt").write_text("REQ-1: widget\n") + (root / "src/widget.py").write_text("value = 1\n") + (root / "README.md").write_text("human documentation\n") + (root / ".pdd/sync-ownership.json").write_text( + json.dumps( + { + "rules": [ + { + "pattern": "README.md", + "inventory": "HUMAN_OWNED", + "role": "documentation", + "owner": "docs@example.com", + } + ] + } + ) + ) + (root / "architecture.json").write_text( + '[{"filename":"widget_python.prompt","filepath":"src/widget.py"}]\n' + ) + base = _commit(root, "protected base") + (root / "prompts/helper_python.prompt").write_text("REQ-1: helper\n") + head = _commit(root, "candidate adds unit") + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + tracked = set(_git(root, "ls-tree", "-r", "--name-only", head).splitlines()) + partition = { + item.candidate_id.artifact_relpath.as_posix() + for item in manifest.candidates + } + if partition != tracked or manifest.unaccounted_tracked_paths: + raise AssertionError("base/head tracked inventory was incomplete") + if len(manifest.expected_managed) != 2: + raise AssertionError("candidate-added managed unit was omitted") + (root / "prompts/widget_python.prompt").unlink() + removed = _commit(root, "candidate removes unit") + deletion = build_unit_manifest(root, base_ref=head, head_ref=removed) + if not any("removed managed unit" in reason for reason in deletion.invalid_reasons): + raise AssertionError("whole-unit deletion did not remain blocking") + return ScenarioResult("complete-base-head-inventory", "PASS") + + +def _run_fixture_profile(content: str) -> EvidenceOutcome: + with tempfile.TemporaryDirectory(prefix="pdd-checker-runner-") as directory: + root = Path(directory) + _git_fixture(root) + (root / "tests").mkdir() + (root / "tests/test_widget.py").write_text(content) + head = _commit(root, "runner fixture") + unit = UnitId( + "checker-fixture", PurePosixPath("prompts/widget_python.prompt"), "python" + ) + profile = _profile(unit) + _envelope, executions = run_profile( + root, + profile, + RunBinding("snapshot", head, head), + AttestationIssue( + AttestationSigner("runner-fixture", b"r" * 32), + "runner-attestation", + "runner-nonce", + datetime.now(timezone.utc), + ), + RunnerConfig(timeout_seconds=30), + ) + return executions[0].outcome + + +def _trusted_runner_outcomes(_arguments: argparse.Namespace) -> ScenarioResult: + expected = { + "def test_widget(): assert True\n": EvidenceOutcome.PASS, + "import pytest\n@pytest.mark.skip(reason='required')\n" + "def test_widget(): pass\n": EvidenceOutcome.SKIP, + "import pytest\n@pytest.mark.xfail(reason='known')\n" + "def test_widget(): assert True\n": EvidenceOutcome.XFAIL, + "value = 1\n": EvidenceOutcome.NOT_COLLECTED, + } + for content, outcome in expected.items(): + observed = _run_fixture_profile(content) + if observed is not outcome: + raise AssertionError(f"runner normalized {observed.value}, expected {outcome.value}") + return ScenarioResult("trusted-runner-outcomes", "PASS") + + +def _merge_base_movement(_arguments: argparse.Namespace) -> ScenarioResult: + with tempfile.TemporaryDirectory(prefix="pdd-checker-merge-") as directory: + root = Path(directory) + _git_fixture(root) + (root / "src").mkdir() + target = root / "src/widget.py" + target.write_text("base = True\n") + _commit(root, "base") + writes = ( + PlannedWrite(PurePosixPath("src/widget.py"), b"repair = True\n", "100644"), + ) + stale = TransactionManager(root) + stale.prepare("stale-repair", writes) + target.write_text("merged = True\n") + _commit(root, "merge group moved") + try: + stale.commit("stale-repair") + except TransactionConflict: + pass + else: + raise AssertionError("stale repair survived merge-group movement") + fresh = TransactionManager(root) + fresh.prepare("fresh-repair", writes) + fresh.commit("fresh-repair") + before = _git(root, "status", "--porcelain", "--untracked-files=all") + second = fresh.prepare("post-merge-second", writes) + after = _git(root, "status", "--porcelain", "--untracked-files=all") + changes = int(before != after) + len(second.changed_paths) + if changes: + raise AssertionError("post-merge immediate rerun changed the tree") + return ScenarioResult( + "merge-group-base-movement-and-stale-repair", + "PASS", + post_merge_tree_changes=0, + ) + + +def _transactional_report(_arguments: argparse.Namespace) -> ScenarioResult: + with tempfile.TemporaryDirectory(prefix="pdd-checker-noop-") as directory: + root = Path(directory) + writes = ( + PlannedWrite(PurePosixPath("src/widget.py"), b"value = 1\n", "100644"), + PlannedWrite(PurePosixPath(".pdd/evidence/widget.json"), b"{}\n", "100644"), + PlannedWrite(PurePosixPath(".pdd/meta/v2/widget.json"), b"{}\n", "100644"), + ) + manager = TransactionManager(root) + manager.prepare("first", writes) + manager.commit("first") + second = manager.prepare("second", writes) + if not second.no_op or second.changed_paths: + raise AssertionError("immediate transaction rerun was not a no-op") + return ScenarioResult( + "transactional-canonical-report", + "PASS", + ) + + +def _built_wheel(_arguments: argparse.Namespace) -> ScenarioResult: + if "site-packages" not in Path(__file__).resolve().parts: + raise AssertionError("scenario harness is not running from an installed wheel") + return ScenarioResult("built-wheel-clean-environment", "PASS") + + +def _cloud_canary(arguments: argparse.Namespace) -> ScenarioResult: + cloud = Path(arguments.cloud_root).resolve() + count = count_vendored_sync_semantics( + cloud, base_ref=arguments.cloud_base_ref, head_ref=arguments.cloud_head_ref + ) + if count: + raise AssertionError(f"pdd_cloud retains {count} vendored sync semantics") + with tempfile.TemporaryDirectory(prefix="pdd-cloud-canary-") as directory: + report = build_canonical_report( + cloud, + CanonicalReportOptions( + base_ref=arguments.cloud_base_ref, + head_ref=arguments.cloud_head_ref, + replay_ledger_path=Path(directory) / "replay.json", + ), + ) + if report.get("ok") is not True: + counts = report.get("counts", {}) + raise AssertionError( + "pdd_cloud canonical report is red: " + + json.dumps(counts, sort_keys=True, separators=(",", ":")) + ) + return ScenarioResult("pdd-cloud-real-consumer-canary", "PASS") + + +def _owned_harness(_arguments: argparse.Namespace) -> ScenarioResult: + if any( + marker in key.upper() + for key in os.environ + for marker in ("SIGNING_KEY", "CERTIFICATE_SIGNING", "ATTESTATION_SIGNING") + ): + raise AssertionError("scenario child received signing material") + return ScenarioResult("released-checker-owned-scenario-harness", "PASS") + + +SCENARIOS: dict[str, Callable[[argparse.Namespace], ScenarioResult]] = { + "source-edit-matrix": _source_edit_matrix, + "missing-corrupt-delete-mode": _missing_corrupt_delete_mode, + "transaction-crash-race-recovery": _transaction_crash_race, + "forged-stale-replayed-revoked-evidence": _evidence_guards, + "complete-base-head-inventory": _complete_inventory, + "trusted-runner-outcomes": _trusted_runner_outcomes, + "transactional-canonical-report": _transactional_report, + "merge-group-base-movement-and-stale-repair": _merge_base_movement, + "built-wheel-clean-environment": _built_wheel, + "pdd-cloud-real-consumer-canary": _cloud_canary, + "released-checker-owned-scenario-harness": _owned_harness, +} + + +def run_scenarios(arguments: argparse.Namespace) -> tuple[ScenarioResult, ...]: + # pylint: disable=broad-exception-caught + """Run every checker-owned scenario and normalize exceptions as failures.""" + results: list[ScenarioResult] = [] + for scenario_id in sorted(REQUIRED_SCENARIOS): + scenario = SCENARIOS.get(scenario_id) + if scenario is None: + results.append(ScenarioResult(scenario_id, "MISSING", "scenario is absent")) + continue + try: + result = scenario(arguments) + except (Exception, SystemExit) as exc: + result = ScenarioResult(scenario_id, "FAIL", f"{type(exc).__name__}: {exc}") + results.append(result) + return tuple(results) + + +def main() -> None: + """Run the complete harness and write a normalized machine result.""" + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cloud-root", required=True) + parser.add_argument("--cloud-base-ref", required=True) + parser.add_argument("--cloud-head-ref", required=True) + arguments = parser.parse_args() + results = run_scenarios(arguments) + payload = { + "schema_version": 1, + "results": [asdict(result) for result in results], + } + arguments.output.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + if any(result.status != "PASS" for result in results): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/pdd/sync_core/snapshot.py b/pdd/sync_core/snapshot.py new file mode 100644 index 000000000..f64556861 --- /dev/null +++ b/pdd/sync_core/snapshot.py @@ -0,0 +1,68 @@ +"""Canonical unit snapshots built from manifests, includes, and profiles.""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath + +from .includes import IncludeGraphError, build_include_closure +from .manifest import ManifestUnit, UnitManifest +from .path_policy import PathPolicy, PathPolicyError +from .types import ArtifactSnapshot, UnitSnapshot, VerificationProfile + + +class SnapshotError(ValueError): + """Raised when a complete policy-valid unit snapshot cannot be built.""" + + +def _obligation_role(kind: str) -> str: + normalized = kind.casefold() + if normalized in {"test", "story", "policy", "requirement"}: + return "test" + if normalized == "example": + return "example" + return "validation" + + +def build_unit_snapshot( + root: Path, + manifest: UnitManifest, + unit: ManifestUnit, + profile: VerificationProfile, +) -> UnitSnapshot: + """Build the complete current snapshot for one present managed unit.""" + if not unit.present_in_head: + raise SnapshotError("cannot snapshot a unit absent from the checked head") + if profile.unit_id != unit.unit_id: + raise SnapshotError("verification profile identity does not match unit") + policy = PathPolicy(root) + artifacts: dict[tuple[str, PurePosixPath], ArtifactSnapshot] = {} + + def add(role: str, relpath: PurePosixPath, required: bool = True) -> None: + snapshot = policy.snapshot(role, relpath, required=required) + artifacts[(role, relpath)] = snapshot + + try: + add("prompt", unit.unit_id.prompt_relpath) + closure = build_include_closure( + unit.unit_id.prompt_relpath, + policy, + root_aliases=unit.artifact_paths, + ) + for included in closure.artifacts: + add("include", included.relpath) + for path in unit.artifact_paths: + add("code", path) + for obligation in profile.obligations: + role = _obligation_role(obligation.kind) + for path in obligation.artifact_paths: + add(role, path, required=obligation.required) + except (FileNotFoundError, PathPolicyError, IncludeGraphError) as exc: + raise SnapshotError(f"cannot build unit snapshot: {exc}") from exc + return UnitSnapshot( + unit.unit_id, + tuple(sorted(artifacts.values())), + manifest.digest(), + closure.digest(), + profile.profile_digest, + closure.has_nondeterministic_query, + ) diff --git a/pdd/sync_core/transaction.py b/pdd/sync_core/transaction.py new file mode 100644 index 000000000..5ff8b304a --- /dev/null +++ b/pdd/sync_core/transaction.py @@ -0,0 +1,510 @@ +"""Crash-durable multi-file synchronization transactions.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import tempfile +from contextlib import ExitStack +from dataclasses import dataclass +from enum import Enum +from pathlib import Path, PurePosixPath +from typing import Callable, Optional + +from filelock import FileLock + +from .durability import fsync_directory +from .path_policy import PathPolicy + + +class TransactionError(RuntimeError): + """Raised when a synchronization transaction cannot proceed safely.""" + + +class TransactionConflict(TransactionError): + """Raised when a destination changes after transaction planning.""" + + +class RecoveryRequired(TransactionError): + """Raised when a durable COMMITTING transaction must be recovered.""" + + +class TransactionPhase(str, Enum): + """Durable WAL state controlling deterministic recovery behavior.""" + + PREPARED = "PREPARED" + COMMITTING = "COMMITTING" + COMMITTED = "COMMITTED" + ROLLED_BACK = "ROLLED_BACK" + + +@dataclass(frozen=True) +class FileState: + """CAS-relevant content, type, and normalized mode for one destination.""" + + exists: bool + digest: str | None + git_mode: str | None + file_type: str + + +@dataclass(frozen=True) +class PlannedWrite: + """One repository-relative regular-file output in a transaction.""" + + relpath: PurePosixPath + content: bytes + git_mode: str + secret: bool = False + expected: FileState | None = None + + def __post_init__(self) -> None: + if self.git_mode not in {"100644", "100755"}: + raise TransactionError("transactions only install regular Git file modes") + + +@dataclass(frozen=True) +class TransactionResult: + """Observable completion state for one mutation attempt.""" + + transaction_id: str + phase: TransactionPhase + changed_paths: tuple[PurePosixPath, ...] + no_op: bool + + +def _digest(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def _git_mode(mode: int) -> str: + executable = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + return "100755" if mode & executable else "100644" + + +def _file_state(path: Path) -> FileState: + if not path.exists() and not path.is_symlink(): + return FileState(False, None, None, "missing") + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + return FileState(True, None, "120000", "symlink") + if not stat.S_ISREG(mode): + return FileState(True, None, None, "special") + content = path.read_bytes() + return FileState(True, _digest(content), _git_mode(mode), "regular") + + +def _state_payload(state: FileState) -> dict[str, object]: + return { + "exists": state.exists, + "digest": state.digest, + "git_mode": state.git_mode, + "file_type": state.file_type, + } + + +def _parse_state(payload: dict[str, object]) -> FileState: + return FileState( + bool(payload["exists"]), + payload.get("digest") if isinstance(payload.get("digest"), str) else None, + payload.get("git_mode") + if isinstance(payload.get("git_mode"), str) + else None, + str(payload["file_type"]), + ) + + +class TransactionManager: + """Prepare, commit, inspect, and recover repository mutation journals.""" + + def __init__(self, checkout_root: Path) -> None: + self.checkout_root = Path(checkout_root).resolve() + self.policy = PathPolicy(self.checkout_root) + self.state_root = self.checkout_root / ".pdd/transactions" + self.lock_root = self.checkout_root / ".pdd/locks/transactions" + + def _ensure_private_directory(self, path: Path) -> None: + current = self.checkout_root + for part in path.relative_to(self.checkout_root).parts: + current = current / part + if current.exists() or current.is_symlink(): + mode = current.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode): + raise TransactionError(f"transaction state path is unsafe: {current}") + else: + current.mkdir(mode=0o700) + if current == path: + os.chmod(current, 0o700) + + def _transaction_dir(self, transaction_id: str) -> Path: + safe_characters = ( + "-_.0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + ) + if not transaction_id or any( + character not in safe_characters for character in transaction_id + ): + raise TransactionError("transaction ID contains unsafe characters") + return self.state_root / transaction_id + + @staticmethod + def _journal_path(transaction_dir: Path) -> Path: + return transaction_dir / "journal.json" + + def _write_journal(self, transaction_dir: Path, payload: dict[str, object]) -> None: + journal_path = self._journal_path(transaction_dir) + descriptor, temporary_name = tempfile.mkstemp( + prefix=".journal.", suffix=".tmp", dir=transaction_dir + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + encoded = json.dumps(payload, sort_keys=True, indent=2).encode() + b"\n" + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, journal_path) + fsync_directory(transaction_dir) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + def _read_journal(self, transaction_dir: Path) -> dict[str, object]: + path = self._journal_path(transaction_dir) + if path.is_symlink() or not path.is_file(): + raise TransactionError(f"transaction journal is missing or unsafe: {path}") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise TransactionError(f"transaction journal is corrupt: {path}") from exc + if not isinstance(payload, dict): + raise TransactionError("transaction journal root must be an object") + return payload + + def _write_blob(self, path: Path, content: bytes, mode: int = 0o600) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + + def _validate_plan(self, writes: tuple[PlannedWrite, ...]) -> None: + if not writes: + raise TransactionError("transaction plan must contain at least one write") + paths = [write.relpath for write in writes] + if len(paths) != len(set(paths)): + raise TransactionError("transaction plan contains duplicate destinations") + if any(write.secret for write in writes): + raise TransactionError( + "secret-labeled rollback content requires configured encryption" + ) + for write in writes: + self.policy.resolve(write.relpath, allow_missing=True) + + def _available_space_check(self, writes: tuple[PlannedWrite, ...]) -> None: + required = sum(len(write.content) for write in writes) * 2 + 1024 * 1024 + if shutil.disk_usage(self.checkout_root).free < required: + raise TransactionError("insufficient space for prepared and rollback state") + + def _entry( + self, + transaction_dir: Path, + index: int, + write: PlannedWrite, + ) -> dict[str, object]: + resolved = self.policy.resolve(write.relpath, allow_missing=True) + before = _file_state(resolved.canonical_path) + if write.expected is not None and before != write.expected: + raise TransactionConflict( + f"destination changed before prepare: {write.relpath}" + ) + if before.file_type not in {"missing", "regular"}: + raise TransactionError(f"destination is not a regular file: {write.relpath}") + prepared_name = f"prepared-{index}.blob" + rollback_name = f"rollback-{index}.blob" if before.exists else None + self._write_blob(transaction_dir / prepared_name, write.content) + if rollback_name: + self._write_blob(transaction_dir / rollback_name, resolved.canonical_path.read_bytes()) + return { + "relpath": write.relpath.as_posix(), + "desired_digest": _digest(write.content), + "desired_mode": write.git_mode, + "precondition": _state_payload(write.expected or before), + "prepared_blob": prepared_name, + "rollback_blob": rollback_name, + "installed": False, + } + + def prepare( + self, + transaction_id: str, + writes: tuple[PlannedWrite, ...], + *, + shared_resources: tuple[PurePosixPath, ...] = (), + ) -> TransactionResult: + """Persist prepared outputs and rollback state without changing destinations.""" + self._validate_plan(writes) + self._available_space_check(writes) + desired_states = { + write.relpath: FileState(True, _digest(write.content), write.git_mode, "regular") + for write in writes + } + current_states = { + write.relpath: _file_state( + self.policy.resolve(write.relpath, allow_missing=True).canonical_path + ) + for write in writes + } + for write in writes: + if write.expected is not None and current_states[write.relpath] != write.expected: + raise TransactionConflict( + f"destination changed before prepare: {write.relpath}" + ) + if current_states == desired_states: + return TransactionResult(transaction_id, TransactionPhase.COMMITTED, (), True) + self._ensure_private_directory(self.state_root) + self._ensure_private_directory(self.lock_root) + transaction_dir = self._transaction_dir(transaction_id) + if transaction_dir.exists(): + raise TransactionError(f"transaction already exists: {transaction_id}") + transaction_dir.mkdir(mode=0o700) + entries = [self._entry(transaction_dir, index, write) for index, write in enumerate(writes)] + payload: dict[str, object] = { + "schema_version": 1, + "transaction_id": transaction_id, + "phase": TransactionPhase.PREPARED.value, + "shared_resources": [path.as_posix() for path in sorted(shared_resources)], + "entries": entries, + } + self._write_journal(transaction_dir, payload) + fsync_directory(self.state_root) + return TransactionResult( + transaction_id, + TransactionPhase.PREPARED, + tuple(write.relpath for write in writes), + False, + ) + + def _locks(self, payload: dict[str, object]) -> ExitStack: + self._ensure_private_directory(self.lock_root) + resources = [str(item) for item in payload.get("shared_resources", [])] + entries = payload.get("entries", []) + if not isinstance(entries, list): + raise TransactionError("transaction entries are malformed") + resources.extend(str(item.get("relpath")) for item in entries if isinstance(item, dict)) + stack = ExitStack() + for resource in sorted(set(resources)): + lock_name = hashlib.sha256(resource.encode()).hexdigest() + ".lock" + stack.enter_context(FileLock(str(self.lock_root / lock_name))) + return stack + + def _install_entry( + self, + transaction_dir: Path, + entry: dict[str, object], + ) -> None: + relpath = PurePosixPath(str(entry["relpath"])) + resolved = self.policy.resolve(relpath, allow_missing=True) + current = _file_state(resolved.canonical_path) + desired = FileState( + True, + str(entry["desired_digest"]), + str(entry["desired_mode"]), + "regular", + ) + if current == desired: + return + precondition_payload = entry.get("precondition") + if not isinstance(precondition_payload, dict): + raise TransactionError("transaction precondition is malformed") + if current != _parse_state(precondition_payload): + raise TransactionConflict(f"destination changed: {relpath}") + resolved.canonical_path.parent.mkdir(parents=True, exist_ok=True) + prepared = transaction_dir / str(entry["prepared_blob"]) + if prepared.is_symlink() or not prepared.is_file(): + raise TransactionError(f"prepared transaction blob is unsafe: {relpath}") + content = prepared.read_bytes() + if _digest(content) != desired.digest: + raise TransactionError(f"prepared transaction blob is corrupt: {relpath}") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{resolved.canonical_path.name}.", + suffix=".pdd-tmp", + dir=resolved.canonical_path.parent, + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o755 if desired.git_mode == "100755" else 0o644) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, resolved.canonical_path) + fsync_directory(resolved.canonical_path.parent) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + def _validate_prepared_entries( + self, transaction_dir: Path, entries: list[object] + ) -> None: + """Validate the complete prepared set before changing any destination.""" + for item in entries: + if not isinstance(item, dict): + raise TransactionError("transaction entry is malformed") + relpath = PurePosixPath(str(item["relpath"])) + prepared = transaction_dir / str(item["prepared_blob"]) + if prepared.is_symlink() or not prepared.is_file(): + raise TransactionError(f"prepared transaction blob is unsafe: {relpath}") + if _digest(prepared.read_bytes()) != str(item["desired_digest"]): + raise TransactionError(f"prepared transaction blob is corrupt: {relpath}") + + def _restore_entries( + self, transaction_dir: Path, entries: list[object] + ) -> None: + """Restore every already-installed destination from durable pre-state.""" + for item in reversed(entries): + if not isinstance(item, dict) or item.get("installed") is not True: + continue + relpath = PurePosixPath(str(item["relpath"])) + destination = self.policy.resolve(relpath, allow_missing=True).canonical_path + precondition = item.get("precondition") + if not isinstance(precondition, dict): + raise TransactionError("transaction rollback precondition is malformed") + before = _parse_state(precondition) + rollback_name = item.get("rollback_blob") + if not before.exists: + destination.unlink(missing_ok=True) + fsync_directory(destination.parent) + continue + rollback = transaction_dir / str(rollback_name) + if rollback.is_symlink() or not rollback.is_file(): + raise TransactionError(f"rollback transaction blob is unsafe: {relpath}") + content = rollback.read_bytes() + if _digest(content) != before.digest: + raise TransactionError(f"rollback transaction blob is corrupt: {relpath}") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", suffix=".pdd-rollback", dir=destination.parent + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o755 if before.git_mode == "100755" else 0o644) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, destination) + fsync_directory(destination.parent) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + def commit( + self, + transaction_id: str, + *, + crash_hook: Optional[Callable[[str], None]] = None, + ) -> TransactionResult: + """Install all prepared writes after a durable COMMITTING marker.""" + transaction_dir = self._transaction_dir(transaction_id) + payload = self._read_journal(transaction_dir) + phase = TransactionPhase(str(payload["phase"])) + if phase is TransactionPhase.COMMITTED: + return TransactionResult(transaction_id, phase, (), True) + if phase is not TransactionPhase.PREPARED: + raise RecoveryRequired(f"pdd recover --transaction {transaction_id}") + hook = crash_hook or (lambda _event: None) + entries = payload.get("entries") + if not isinstance(entries, list): + raise TransactionError("transaction entries are malformed") + with self._locks(payload): + self._validate_prepared_entries(transaction_dir, entries) + for item in entries: + if not isinstance(item, dict): + raise TransactionError("transaction entry is malformed") + relpath = PurePosixPath(str(item["relpath"])) + current = _file_state( + self.policy.resolve(relpath, allow_missing=True).canonical_path + ) + precondition = item.get("precondition") + if not isinstance(precondition, dict) or current != _parse_state(precondition): + raise TransactionConflict(f"destination changed: {relpath}") + payload["phase"] = TransactionPhase.COMMITTING.value + self._write_journal(transaction_dir, payload) + hook("after_committing") + changed: list[PurePosixPath] = [] + try: + for index, item in enumerate(entries): + if not isinstance(item, dict): + raise TransactionError("transaction entry is malformed") + self._install_entry(transaction_dir, item) + item["installed"] = True + changed.append(PurePosixPath(str(item["relpath"]))) + self._write_journal(transaction_dir, payload) + hook(f"after_install:{index}") + except TransactionError: + self._restore_entries(transaction_dir, entries) + payload["phase"] = TransactionPhase.ROLLED_BACK.value + self._write_journal(transaction_dir, payload) + raise + payload["phase"] = TransactionPhase.COMMITTED.value + self._write_journal(transaction_dir, payload) + hook("after_commit") + return TransactionResult( + transaction_id, TransactionPhase.COMMITTED, tuple(changed), False + ) + + def incomplete(self) -> tuple[str, ...]: + """List transactions requiring explicit recovery without changing bytes.""" + if not self.state_root.exists(): + return () + pending: list[str] = [] + for transaction_dir in sorted(self.state_root.iterdir()): + if not transaction_dir.is_dir() or transaction_dir.is_symlink(): + continue + phase = TransactionPhase(str(self._read_journal(transaction_dir)["phase"])) + if phase not in {TransactionPhase.COMMITTED, TransactionPhase.ROLLED_BACK}: + pending.append(transaction_dir.name) + return tuple(pending) + + def recover(self, transaction_id: str) -> TransactionResult: + """Complete COMMITTING work or discard a PREPARED transaction idempotently.""" + transaction_dir = self._transaction_dir(transaction_id) + payload = self._read_journal(transaction_dir) + phase = TransactionPhase(str(payload["phase"])) + if phase is TransactionPhase.COMMITTED: + return TransactionResult(transaction_id, phase, (), True) + if phase is TransactionPhase.PREPARED: + payload["phase"] = TransactionPhase.ROLLED_BACK.value + self._write_journal(transaction_dir, payload) + return TransactionResult( + transaction_id, TransactionPhase.ROLLED_BACK, (), False + ) + if phase is not TransactionPhase.COMMITTING: + return TransactionResult(transaction_id, phase, (), True) + entries = payload.get("entries") + if not isinstance(entries, list): + raise TransactionError("transaction entries are malformed") + changed: list[PurePosixPath] = [] + with self._locks(payload): + try: + self._validate_prepared_entries(transaction_dir, entries) + for item in entries: + if not isinstance(item, dict): + raise TransactionError("transaction entry is malformed") + self._install_entry(transaction_dir, item) + item["installed"] = True + changed.append(PurePosixPath(str(item["relpath"]))) + self._write_journal(transaction_dir, payload) + except TransactionError: + self._restore_entries(transaction_dir, entries) + payload["phase"] = TransactionPhase.ROLLED_BACK.value + self._write_journal(transaction_dir, payload) + raise + payload["phase"] = TransactionPhase.COMMITTED.value + self._write_journal(transaction_dir, payload) + return TransactionResult( + transaction_id, TransactionPhase.COMMITTED, tuple(changed), False + ) diff --git a/pdd/sync_core/trust.py b/pdd/sync_core/trust.py new file mode 100644 index 000000000..229c97cdd --- /dev/null +++ b/pdd/sync_core/trust.py @@ -0,0 +1,433 @@ +"""Attestation issuance and protected trust-policy verification.""" + +from __future__ import annotations + +import base64 +import json +import os +import stat +import tempfile +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Mapping, Optional + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + Ed25519PublicKey, +) +from filelock import FileLock + +from .durability import fsync_directory +from .types import EvidenceOutcome, ObligationEvidence, UnitId + + +class AttestationError(ValueError): + """Raised when semantic evidence fails protected trust policy.""" + + +class ReplayStore: + """Interface for atomic cross-verifier attestation nonce consumption.""" + + def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: + """Record a nonce exactly once or raise AttestationError.""" + raise NotImplementedError + + def is_durable(self) -> bool: + """Return whether nonce history survives verifier process exit.""" + raise NotImplementedError + + +class InMemoryReplayStore(ReplayStore): + """Process-local replay store for isolated tests and one-shot validation.""" + + def __init__(self) -> None: + self._seen: dict[tuple[str, str], str] = {} + + def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: + """Reject nonce reuse for a different signed statement.""" + key = (issuer, nonce) + previous = self._seen.get(key) + if previous is not None: + raise AttestationError("attestation nonce was replayed") + self._seen[key] = attestation_id + + def is_durable(self) -> bool: + """Return false for process-local test storage.""" + return False + + +class FileReplayStore(ReplayStore): + """Locked durable nonce ledger located outside candidate-controlled state.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path).resolve() + self.lock_path = self.path.with_suffix(self.path.suffix + ".lock") + + def _ensure_parent(self) -> None: + parent = self.path.parent + if parent.exists() and (parent.is_symlink() or not parent.is_dir()): + raise AttestationError("replay ledger parent is unsafe") + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(parent, 0o700) + + def _load(self) -> dict[str, str]: + if not self.path.exists(): + return {} + if self.path.is_symlink() or not self.path.is_file(): + raise AttestationError("replay ledger is unsafe") + try: + payload = json.loads(self.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise AttestationError("replay ledger is corrupt") from exc + if not isinstance(payload, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in payload.items() + ): + raise AttestationError("replay ledger has invalid records") + return payload + + def _write(self, payload: dict[str, str]) -> None: + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{self.path.name}.", suffix=".tmp", dir=self.path.parent + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, sort_keys=True, indent=2) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, self.path) + fsync_directory(self.path.parent) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: + """Atomically reject and record every repeated issuer/nonce pair.""" + self._ensure_parent() + with FileLock(str(self.lock_path)): + records = self._load() + key = base64.b64encode(f"{issuer}\0{nonce}".encode()).decode("ascii") + previous = records.get(key) + if previous is not None: + raise AttestationError("attestation nonce was replayed") + records[key] = attestation_id + self._write(records) + if stat.S_IMODE(self.path.stat().st_mode) != 0o600: + raise AttestationError("replay ledger permissions are unsafe") + + def is_durable(self) -> bool: + """Return true for the fsynced external ledger.""" + return True + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None: + raise ValueError("attestation timestamps must be timezone-aware") + return value.astimezone(timezone.utc).isoformat() + + +def _parse_timestamp(value: str) -> datetime: + try: + parsed = datetime.fromisoformat(value) + except ValueError as exc: + raise AttestationError("attestation timestamp is malformed") from exc + if parsed.tzinfo is None: + raise AttestationError("attestation timestamp must include a timezone") + return parsed.astimezone(timezone.utc) + + +@dataclass(frozen=True) +class AttestationBinding: + """Complete subject, input, runner, and Git closure for an attestation.""" + + subject: UnitId + snapshot_digest: str + profile_digest: str + runner_digest: str + tool_version: str + base_sha: str + checked_sha: str + + +@dataclass(frozen=True) +class AttestationValidity: + """Freshness and replay fields covered by the signature.""" + + issued_at: str + expires_at: str + nonce: str + + +@dataclass(frozen=True) +class AttestationEnvelope: + """Signed statement covering one unit's complete verification execution.""" + + attestation_id: str + issuer: str + binding: AttestationBinding + results: tuple[ObligationEvidence, ...] + validity: AttestationValidity + signature: str = "" + + def payload(self) -> bytes: + """Return canonical signed bytes, excluding the signature itself.""" + data = { + "attestation_id": self.attestation_id, + "issuer": self.issuer, + "binding": { + "subject": { + "repository_id": self.binding.subject.repository_id, + "prompt_relpath": self.binding.subject.prompt_relpath.as_posix(), + "language_id": self.binding.subject.language_id, + }, + "snapshot_digest": self.binding.snapshot_digest, + "profile_digest": self.binding.profile_digest, + "runner_digest": self.binding.runner_digest, + "tool_version": self.binding.tool_version, + "base_sha": self.binding.base_sha, + "checked_sha": self.binding.checked_sha, + }, + "results": [ + { + "obligation_id": result.obligation_id, + "outcome": result.outcome.value, + } + for result in sorted(self.results) + ], + "validity": { + "issued_at": self.validity.issued_at, + "expires_at": self.validity.expires_at, + "nonce": self.validity.nonce, + }, + } + return json.dumps(data, sort_keys=True, separators=(",", ":")).encode() + + +@dataclass(frozen=True) +class AttestationRequest: + """Inputs supplied by a trusted runner when issuing evidence.""" + + attestation_id: str + binding: AttestationBinding + results: tuple[ObligationEvidence, ...] + nonce: str + issued_at: Optional[datetime] = None + lifetime: timedelta = timedelta(hours=1) + + +@dataclass(frozen=True) +class EvidenceClaims: + """Claims retained after signature and policy verification.""" + + subject: UnitId + snapshot_digest: str + profile_digest: str + results: tuple[ObligationEvidence, ...] + attestation_id: str + issuer: str + + +_EVIDENCE_SEAL = object() + + +@dataclass(frozen=True, init=False) +class ValidationEvidence: + """Evidence that can only be constructed with the module verifier seal.""" + + claims: EvidenceClaims + + def __init__(self, claims: EvidenceClaims, *, _seal: object) -> None: + if _seal is not _EVIDENCE_SEAL: + raise TypeError("ValidationEvidence must be issued by AttestationTrustPolicy") + object.__setattr__(self, "claims", claims) + + @property + def subject(self) -> UnitId: + """Return the verified subject identity.""" + return self.claims.subject + + @property + def snapshot_digest(self) -> str: + """Return the verified snapshot closure digest.""" + return self.claims.snapshot_digest + + @property + def profile_digest(self) -> str: + """Return the verified profile digest.""" + return self.claims.profile_digest + + @property + def attestation_id(self) -> str: + """Return the verified attestation identifier.""" + return self.claims.attestation_id + + def result_map(self) -> dict[str, EvidenceOutcome]: + """Index normalized outcomes by obligation identifier.""" + return {result.obligation_id: result.outcome for result in self.claims.results} + + +class AttestationSigner: + """Trusted-runner helper that signs verification results.""" + + def __init__(self, issuer: str, key: bytes) -> None: + if not issuer or len(key) != 32: + raise ValueError("attestation signer requires issuer and key") + self.issuer = issuer + self._key = Ed25519PrivateKey.from_private_bytes(key) + + def public_key_bytes(self) -> bytes: + """Return the raw public key suitable for protected trust policy.""" + return self._key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + + def sign_bytes(self, payload: bytes) -> str: + """Return a base64 Ed25519 signature for arbitrary canonical bytes.""" + return base64.b64encode(self._key.sign(payload)).decode("ascii") + + def sign(self, envelope: AttestationEnvelope) -> AttestationEnvelope: + """Attach an HMAC signature to an otherwise complete envelope.""" + signature = self.sign_bytes(envelope.payload()) + return replace(envelope, signature=signature) + + def issue(self, request: AttestationRequest) -> AttestationEnvelope: + """Create a signed envelope bound to one checked commit and base.""" + issued = request.issued_at or datetime.now(timezone.utc) + validity = AttestationValidity( + _timestamp(issued), + _timestamp(issued + request.lifetime), + request.nonce, + ) + return self.sign( + AttestationEnvelope( + request.attestation_id, + self.issuer, + request.binding, + request.results, + validity, + ) + ) + + +class AttestationTrustPolicy: + """Protected issuer, freshness, revocation, and replay policy.""" + + def __init__( + self, + trusted_issuers: Mapping[str, bytes], + *, + revoked_issuers: frozenset[str] = frozenset(), + revoked_attestations: frozenset[str] = frozenset(), + clock_skew: timedelta = timedelta(minutes=5), + maximum_lifetime: timedelta = timedelta(hours=1), + replay_store: ReplayStore | None = None, + ) -> None: + # pylint: disable=too-many-arguments + self._trusted_issuers = dict(trusted_issuers) + self._revoked_issuers = revoked_issuers + self._revoked_attestations = revoked_attestations + self._clock_skew = clock_skew + self._maximum_lifetime = maximum_lifetime + self._replay_store = replay_store or InMemoryReplayStore() + + def reset_replay_cache(self) -> None: + """Reject unsafe attempts to erase replay history through the verifier.""" + raise AttestationError("replay history cannot be reset by candidate code") + + def _verify_signature(self, envelope: AttestationEnvelope) -> None: + key_bytes = self._trusted_issuers.get(envelope.issuer) + if key_bytes is None: + raise AttestationError("attestation issuer is not trusted") + try: + signature = base64.b64decode(envelope.signature, validate=True) + Ed25519PublicKey.from_public_bytes(key_bytes).verify( + signature, envelope.payload() + ) + except (ValueError, InvalidSignature) as exc: + raise AttestationError("attestation signature is invalid") from exc + + def _verify_freshness(self, envelope: AttestationEnvelope, now: datetime) -> None: + issued = _parse_timestamp(envelope.validity.issued_at) + expires = _parse_timestamp(envelope.validity.expires_at) + if expires <= issued: + raise AttestationError("attestation validity interval is invalid") + if expires - issued > self._maximum_lifetime: + raise AttestationError("attestation validity interval exceeds policy") + if issued > now + self._clock_skew: + raise AttestationError("attestation is not yet valid") + if expires <= now: + raise AttestationError("attestation is expired") + + def _verify_replay(self, envelope: AttestationEnvelope) -> None: + self._replay_store.consume( + envelope.issuer, envelope.validity.nonce, envelope.attestation_id + ) + + def _verify( + self, + envelope: AttestationEnvelope, + expected: AttestationBinding, + *, + now: Optional[datetime] = None, + consume_replay: bool, + ) -> ValidationEvidence: + if envelope.issuer in self._revoked_issuers: + raise AttestationError("attestation issuer is revoked") + if envelope.attestation_id in self._revoked_attestations: + raise AttestationError("attestation is revoked") + self._verify_signature(envelope) + self._verify_freshness(envelope, now or datetime.now(timezone.utc)) + if envelope.binding != expected: + raise AttestationError("attestation does not match the checked closure") + identifiers = [result.obligation_id for result in envelope.results] + if len(identifiers) != len(set(identifiers)): + raise AttestationError("attestation contains duplicate obligation results") + if not envelope.binding.runner_digest or not envelope.binding.tool_version: + raise AttestationError("attestation runner identity is incomplete") + if consume_replay: + self._verify_replay(envelope) + claims = EvidenceClaims( + envelope.binding.subject, + envelope.binding.snapshot_digest, + envelope.binding.profile_digest, + envelope.results, + envelope.attestation_id, + envelope.issuer, + ) + return ValidationEvidence(claims, _seal=_EVIDENCE_SEAL) + + def verify( + self, + envelope: AttestationEnvelope, + expected: AttestationBinding, + *, + now: Optional[datetime] = None, + ) -> ValidationEvidence: + """Verify all protected bindings and consume evidence exactly once.""" + return self._verify( + envelope, expected, now=now, consume_replay=True + ) + + def verify_current_for_idempotency( + self, + envelope: AttestationEnvelope, + expected: AttestationBinding, + *, + now: Optional[datetime] = None, + ) -> ValidationEvidence: + """Recheck current trusted state without claiming a new evidence use.""" + return self._verify( + envelope, expected, now=now, consume_replay=False + ) + + +def passing_result(obligation_id: str) -> ObligationEvidence: + """Build a normalized PASS result for a trusted runner adapter.""" + return ObligationEvidence(obligation_id, EvidenceOutcome.PASS) diff --git a/pdd/sync_core/types.py b/pdd/sync_core/types.py new file mode 100644 index 000000000..e494b4798 --- /dev/null +++ b/pdd/sync_core/types.py @@ -0,0 +1,330 @@ +"""Immutable value types shared by every synchronization consumer.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from enum import Enum +from pathlib import PurePosixPath +from typing import Optional, Union + + +class InventoryStatus(str, Enum): + """Intrinsic ownership classification for a discovered subject.""" + + MANAGED = "MANAGED" + HUMAN_OWNED = "HUMAN_OWNED" + REMOVED = "REMOVED" + INVALID = "INVALID" + + +class BaselineStatus(str, Enum): + """Relationship between current bytes and the trusted baseline.""" + + CURRENT = "CURRENT" + DRIFTED = "DRIFTED" + UNBASELINED = "UNBASELINED" + CORRUPT = "CORRUPT" + + +class SemanticStatus(str, Enum): + """Current semantic agreement backed by validation evidence.""" + + VERIFIED = "VERIFIED" + UNKNOWN = "UNKNOWN" + CONFLICT = "CONFLICT" + FAILED = "FAILED" + + +class EvidenceOutcome(str, Enum): + """Normalized terminal result for a verification obligation.""" + + PASS = "PASS" + FAIL = "FAIL" + ERROR = "ERROR" + COLLECTION_ERROR = "COLLECTION_ERROR" + SKIP = "SKIP" + XFAIL = "XFAIL" + XPASS = "XPASS" + DESELECTED = "DESELECTED" + TIMEOUT = "TIMEOUT" + NOT_COLLECTED = "NOT_COLLECTED" + QUARANTINED = "QUARANTINED" + FLAKY = "FLAKY" + + +def _validate_repository_id(repository_id: str) -> None: + if not repository_id or repository_id.strip() != repository_id: + raise ValueError("repository_id must be a non-empty canonical identifier") + + +def _validate_relpath(path: PurePosixPath) -> None: + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError(f"path must be repository-relative: {path}") + + +@dataclass(frozen=True, order=True) +class CandidateId: + """Stable identity for an artifact not yet adopted into a unit.""" + + repository_id: str + artifact_relpath: PurePosixPath + role: str + + def __post_init__(self) -> None: + _validate_repository_id(self.repository_id) + _validate_relpath(self.artifact_relpath) + if not self.role: + raise ValueError("candidate role must not be empty") + + +@dataclass(frozen=True, order=True) +class UnitId: + """Stable prompt-backed identity independent of checkout location.""" + + repository_id: str + prompt_relpath: PurePosixPath + language_id: str + + def __post_init__(self) -> None: + _validate_repository_id(self.repository_id) + _validate_relpath(self.prompt_relpath) + if not self.language_id: + raise ValueError("language_id must not be empty") + + +SubjectId = Union[CandidateId, UnitId] + + +@dataclass(frozen=True, order=True) +class ArtifactSnapshot: + """Policy-relevant state of one artifact at an instant.""" + + role: str + relpath: PurePosixPath + digest: Optional[str] + git_mode: Optional[str] + required: bool = True + + def __post_init__(self) -> None: + if not self.role: + raise ValueError("artifact role must not be empty") + _validate_relpath(self.relpath) + if self.digest is not None and not self.digest: + raise ValueError("artifact digest must be non-empty or None") + if self.git_mode not in {None, "100644", "100755", "120000"}: + raise ValueError(f"unsupported normalized Git mode: {self.git_mode}") + + @property + def exists(self) -> bool: + """Return whether both content and artifact mode are present.""" + return self.digest is not None and self.git_mode is not None + + +@dataclass(frozen=True) +class UnitSnapshot: + """Complete hashable state consumed by the pure classifier.""" + + unit_id: UnitId + artifacts: tuple[ArtifactSnapshot, ...] + manifest_digest: str + dependency_snapshot_digest: str + verification_profile_digest: str + nondeterministic_inputs: bool = False + + def __post_init__(self) -> None: + identities = [(artifact.role, artifact.relpath) for artifact in self.artifacts] + if len(identities) != len(set(identities)): + raise ValueError("artifact role/path identities must be unique") + for value in ( + self.manifest_digest, + self.dependency_snapshot_digest, + self.verification_profile_digest, + ): + if not value: + raise ValueError("snapshot closure digests must not be empty") + + def artifact_map(self) -> dict[tuple[str, PurePosixPath], ArtifactSnapshot]: + """Index artifacts by role and repository-relative path.""" + return {(artifact.role, artifact.relpath): artifact for artifact in self.artifacts} + + def digest(self) -> str: + """Return a deterministic digest of the complete snapshot closure.""" + payload = { + "unit_id": { + "repository_id": self.unit_id.repository_id, + "prompt_relpath": self.unit_id.prompt_relpath.as_posix(), + "language_id": self.unit_id.language_id, + }, + "artifacts": [ + { + "role": item.role, + "path": item.relpath.as_posix(), + "digest": item.digest, + "git_mode": item.git_mode, + "required": item.required, + } + for item in sorted(self.artifacts) + ], + "manifest_digest": self.manifest_digest, + "dependency_snapshot_digest": self.dependency_snapshot_digest, + "verification_profile_digest": self.verification_profile_digest, + "nondeterministic_inputs": self.nondeterministic_inputs, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +@dataclass(frozen=True, order=True) +class VerificationObligation: + """One protected requirement that must receive trusted evidence.""" + + obligation_id: str + kind: str + validator_id: str + validator_config_digest: str + requirement_ids: tuple[str, ...] + artifact_paths: tuple[PurePosixPath, ...] + required: bool = True + + def __post_init__(self) -> None: + if not self.obligation_id or not self.kind or not self.validator_id: + raise ValueError("verification obligation identity must be complete") + if not self.validator_config_digest: + raise ValueError("validator config digest must not be empty") + for path in self.artifact_paths: + _validate_relpath(path) + + +@dataclass(frozen=True) +class VerificationProfile: + """Protected set of obligations for a unit.""" + + unit_id: UnitId + obligations: tuple[VerificationObligation, ...] + required_requirement_ids: tuple[str, ...] + profile_digest: str + + def __post_init__(self) -> None: + obligation_ids = [item.obligation_id for item in self.obligations] + if len(obligation_ids) != len(set(obligation_ids)): + raise ValueError("verification profile contains duplicate obligation IDs") + if len(self.required_requirement_ids) != len(set(self.required_requirement_ids)): + raise ValueError("verification profile contains duplicate requirement IDs") + if not self.profile_digest: + raise ValueError("verification profile digest must not be empty") + + @property + def complete(self) -> bool: + """Return whether required obligations cover every declared requirement.""" + required_obligations = tuple(item for item in self.obligations if item.required) + if not required_obligations or not self.required_requirement_ids: + return False + covered = { + requirement_id + for item in required_obligations + for requirement_id in item.requirement_ids + } + return set(self.required_requirement_ids).issubset(covered) + + +@dataclass(frozen=True, order=True) +class ObligationEvidence: + """Trusted-checker result for one obligation.""" + + obligation_id: str + outcome: EvidenceOutcome + + +@dataclass(frozen=True) +# pylint: disable=too-many-instance-attributes +class FingerprintProvenance: + """Auditable origin of one canonical fingerprint transition.""" + + kind: str + command: str + transaction_id: str + git_commit: str + timestamp: str + pdd_version: str + reviewed_by: Optional[str] = None + reason: Optional[str] = None + + +@dataclass(frozen=True) +class FingerprintRecord: + """Versioned baseline snapshot with transaction provenance.""" + + snapshot: UnitSnapshot + schema_version: int + hash_algorithm_version: int + provenance: FingerprintProvenance + claimed_semantic_status: SemanticStatus + attestation_ref: Optional[str] + + @property + def valid(self) -> bool: + """Return whether the record uses the authoritative v2 schema.""" + return ( + self.schema_version == 2 + and self.hash_algorithm_version == 2 + and bool(self.provenance.kind) + and bool(self.provenance.command) + and bool(self.provenance.transaction_id) + and bool(self.provenance.git_commit) + and bool(self.provenance.timestamp) + and bool(self.provenance.pdd_version) + ) + + +@dataclass(frozen=True) +class VerdictDetails: + """Diagnostic details kept separate from the core status dimensions.""" + + changed_roles: tuple[str, ...] + reason: str + required_artifacts_missing: tuple[str, ...] = () + evidence_complete: bool = False + + +@dataclass(frozen=True) +class SyncVerdict: + """Independent inventory, baseline, and semantic classifications.""" + + subject: SubjectId + inventory: InventoryStatus + baseline: BaselineStatus + semantic: SemanticStatus + details: VerdictDetails + + @property + def changed_roles(self) -> tuple[str, ...]: + """Return roles that differ from the baseline.""" + return self.details.changed_roles + + @property + def reason(self) -> str: + """Return the stable human-readable verdict explanation.""" + return self.details.reason + + @property + def required_artifacts_missing(self) -> tuple[str, ...]: + """Return required artifact roles absent from the current snapshot.""" + return self.details.required_artifacts_missing + + @property + def evidence_complete(self) -> bool: + """Return whether trusted evidence covers every required obligation.""" + return self.details.evidence_complete + + @property + def in_sync(self) -> bool: + """Return true only for the complete trusted managed predicate.""" + return ( + self.inventory is InventoryStatus.MANAGED + and self.baseline is BaselineStatus.CURRENT + and self.semantic is SemanticStatus.VERIFIED + and not self.required_artifacts_missing + and self.evidence_complete + ) diff --git a/pdd/sync_core/verification.py b/pdd/sync_core/verification.py new file mode 100644 index 000000000..36e8f1b09 --- /dev/null +++ b/pdd/sync_core/verification.py @@ -0,0 +1,246 @@ +"""Protected base/head verification-profile loading and completeness checks.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + +from .manifest import UnitManifest +from .git_io import read_git_blob +from .types import UnitId, VerificationObligation, VerificationProfile + + +PROFILE_PATH = PurePosixPath(".pdd/verification-profiles.json") + + +class VerificationProfileError(ValueError): + """Raised when protected verification-profile data cannot be parsed.""" + + +@dataclass(frozen=True) +class ProfileSet: + """Effective protected profiles and policy violations for a checked head.""" + + profiles: tuple[VerificationProfile, ...] + invalid_reasons: tuple[str, ...] + + @property + def coverage(self) -> float: + """Return the fraction of expected profiles with complete obligations.""" + if not self.profiles: + return 0.0 + complete = sum(profile.complete for profile in self.profiles) + return complete / len(self.profiles) + + def for_unit(self, unit_id: UnitId) -> VerificationProfile | None: + """Return one effective profile by stable unit identity.""" + return next((item for item in self.profiles if item.unit_id == unit_id), None) + + +@dataclass(frozen=True) +class _ProfileInput: + """Parsed requirements and obligations from one immutable Git tree.""" + + requirements: tuple[str, ...] + obligations: tuple[VerificationObligation, ...] + + +_REQUIREMENT_ID = re.compile(r"\bREQ-[A-Za-z0-9_.:-]+\b") + + +def _prompt_requirements(raw: bytes) -> tuple[str, ...]: + """Derive the protected requirement universe from exact prompt bytes.""" + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise VerificationProfileError("profile prompt is not UTF-8") from exc + explicit = tuple(sorted(set(_REQUIREMENT_ID.findall(text)))) + if explicit: + return explicit + return (f"CONTRACT-SHA256:{hashlib.sha256(raw).hexdigest()}",) + + +def _obligation(payload: Mapping[str, Any]) -> VerificationObligation: + try: + requirement_ids = payload["requirement_ids"] + if not isinstance(requirement_ids, list) or not all( + isinstance(item, str) for item in requirement_ids + ): + raise TypeError("requirement_ids must be a string list") + return VerificationObligation( + str(payload["obligation_id"]), + str(payload["kind"]), + str(payload["validator_id"]), + str(payload["validator_config_digest"]), + tuple(sorted(requirement_ids)), + tuple( + sorted( + PurePosixPath(item) + for item in payload.get("artifact_paths", []) + if isinstance(item, str) + ) + ), + bool(payload.get("required", True)), + ) + except (KeyError, TypeError) as exc: + raise VerificationProfileError("verification obligation is malformed") from exc + + +def _load_inputs( + root: Path, + ref: str, + repository_id: str, +) -> tuple[dict[UnitId, _ProfileInput], list[str]]: + # pylint: disable=too-many-branches,too-many-locals + raw = read_git_blob(root, ref, PROFILE_PATH) + if raw is None: + return {}, [] + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise VerificationProfileError(f"{ref}: profile file is malformed") from exc + rows = payload.get("profiles") if isinstance(payload, dict) else None + if not isinstance(rows, list): + raise VerificationProfileError(f"{ref}: profiles must be a list") + profiles: dict[UnitId, _ProfileInput] = {} + invalid: list[str] = [] + for row in rows: + if not isinstance(row, dict): + invalid.append(f"{ref}: profile entry is not an object") + continue + try: + unit_id = UnitId( + repository_id, + PurePosixPath(str(row["prompt_path"])), + str(row["language_id"]), + ) + requirements = row["required_requirement_ids"] + obligations = row["obligations"] + if not isinstance(requirements, list) or not all( + isinstance(item, str) for item in requirements + ): + raise TypeError("required requirements must be a string list") + if not isinstance(obligations, list): + raise TypeError("obligations must be a list") + parsed = _ProfileInput( + tuple(sorted(requirements)), + tuple(sorted(_obligation(item) for item in obligations)), + ) + except (KeyError, TypeError, VerificationProfileError) as exc: + invalid.append(f"{ref}: invalid profile entry: {exc}") + continue + prompt_raw = read_git_blob(root, ref, unit_id.prompt_relpath) + if prompt_raw is None: + invalid.append(f"{ref}: profile prompt is absent: {unit_id.prompt_relpath}") + continue + try: + protected_requirements = _prompt_requirements(prompt_raw) + except VerificationProfileError as exc: + invalid.append(f"{ref}: {unit_id.prompt_relpath}: {exc}") + continue + if parsed.requirements != protected_requirements: + invalid.append( + f"{ref}: {unit_id.prompt_relpath}: profile requirements do not " + "match immutable prompt requirements" + ) + continue + if unit_id in profiles: + invalid.append(f"{ref}: duplicate profile for {unit_id.prompt_relpath}") + else: + profiles[unit_id] = parsed + return profiles, invalid + + +def _profile_digest( + unit_id: UnitId, + requirements: tuple[str, ...], + obligations: tuple[VerificationObligation, ...], +) -> str: + payload = { + "unit": { + "repository_id": unit_id.repository_id, + "prompt_relpath": unit_id.prompt_relpath.as_posix(), + "language_id": unit_id.language_id, + }, + "required_requirement_ids": requirements, + "obligations": [ + { + "obligation_id": item.obligation_id, + "kind": item.kind, + "validator_id": item.validator_id, + "validator_config_digest": item.validator_config_digest, + "requirement_ids": item.requirement_ids, + "artifact_paths": [path.as_posix() for path in item.artifact_paths], + "required": item.required, + } + for item in obligations + ], + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _effective_profile( + unit_id: UnitId, + base: _ProfileInput | None, + head: _ProfileInput | None, +) -> tuple[VerificationProfile, list[str]]: + invalid: list[str] = [] + base_requirements = set(base.requirements if base else ()) + head_requirements = set(head.requirements if head else ()) + if base_requirements - head_requirements: + invalid.append(f"{unit_id.prompt_relpath}: candidate removed protected requirements") + requirements = tuple(sorted(base_requirements | head_requirements)) + base_obligations = {item.obligation_id: item for item in (base.obligations if base else ())} + head_obligations = {item.obligation_id: item for item in (head.obligations if head else ())} + effective = dict(base_obligations) + for obligation_id, obligation in head_obligations.items(): + protected = base_obligations.get(obligation_id) + if protected is not None and protected != obligation: + invalid.append( + f"{unit_id.prompt_relpath}: candidate changed protected obligation " + f"{obligation_id}" + ) + continue + effective[obligation_id] = obligation + invalid.extend( + f"{unit_id.prompt_relpath}: candidate removed protected obligation {item}" + for item in sorted(set(base_obligations) - set(head_obligations)) + ) + obligations = tuple(sorted(effective.values())) + profile = VerificationProfile( + unit_id, + obligations, + requirements, + _profile_digest(unit_id, requirements, obligations), + ) + if not profile.complete: + invalid.append(f"{unit_id.prompt_relpath}: verification profile is incomplete") + return profile, invalid + + +def load_verification_profiles(root: Path, manifest: UnitManifest) -> ProfileSet: + """Load the protected base/candidate union for every expected-managed unit.""" + base, base_invalid = _load_inputs(root, manifest.base_ref, manifest.repository_id) + head, head_invalid = _load_inputs(root, manifest.head_ref, manifest.repository_id) + invalid = base_invalid + head_invalid + profiles: list[VerificationProfile] = [] + expected = set(manifest.expected_managed) + unknown = (set(base) | set(head)) - expected + invalid.extend( + f"profile references non-expected unit {item.prompt_relpath}" + for item in sorted(unknown) + ) + for unit_id in manifest.expected_managed: + if unit_id not in base and unit_id not in head: + invalid.append(f"{unit_id.prompt_relpath}: verification profile is missing") + profile, profile_invalid = _effective_profile( + unit_id, base.get(unit_id), head.get(unit_id) + ) + profiles.append(profile) + invalid.extend(profile_invalid) + return ProfileSet(tuple(profiles), tuple(sorted(set(invalid)))) diff --git a/pdd/sync_core/waivers.py b/pdd/sync_core/waivers.py new file mode 100644 index 000000000..7b5e75ff8 --- /dev/null +++ b/pdd/sync_core/waivers.py @@ -0,0 +1,133 @@ +"""Protected waiver loading for canonical synchronization reports.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path, PurePosixPath +from typing import Any + +from .git_io import read_git_blob +from .manifest import UnitManifest + + +WAIVER_PATH = PurePosixPath(".pdd/sync-waivers.json") + + +@dataclass(frozen=True, order=True) +class SyncWaiver: + """Reviewed, expiring exception bound to one exact managed snapshot.""" + + waiver_id: str + prompt_path: PurePosixPath + snapshot_digest: str + approved_by: str + reason: str + expires_at: datetime + + +@dataclass(frozen=True) +class WaiverSet: + """Effective managed waivers and fail-closed policy violations.""" + + waivers: tuple[SyncWaiver, ...] + invalid_reasons: tuple[str, ...] + + @property + def managed_count(self) -> int: + """Return the gross number of effective managed exceptions.""" + return len(self.waivers) + + +def _waiver(row: Any) -> SyncWaiver: + if not isinstance(row, dict): + raise ValueError("waiver entry is not an object") + try: + expires_at = datetime.fromisoformat(str(row["expires_at"])) + waiver = SyncWaiver( + str(row["waiver_id"]).strip(), + PurePosixPath(str(row["prompt_path"])), + str(row["snapshot_digest"]).strip(), + str(row["approved_by"]).strip(), + str(row["reason"]).strip(), + expires_at, + ) + except (KeyError, ValueError) as exc: + raise ValueError("waiver entry is malformed") from exc + invalid_fields = ( + not waiver.waiver_id, + waiver.prompt_path.is_absolute(), + ".." in waiver.prompt_path.parts, + len(waiver.snapshot_digest) != 64, + any(character not in "0123456789abcdef" for character in waiver.snapshot_digest), + not waiver.approved_by, + not waiver.reason, + waiver.expires_at.tzinfo is None, + ) + if any(invalid_fields): + raise ValueError("waiver entry has invalid protected fields") + return waiver + + +def _load(root: Path, ref: str) -> tuple[dict[str, SyncWaiver], list[str]]: + raw = read_git_blob(root, ref, WAIVER_PATH) + if raw is None: + return {}, [] + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + return {}, [f"{ref}: sync waiver file is malformed: {exc}"] + rows = payload.get("waivers") if isinstance(payload, dict) else None + if not isinstance(rows, list): + return {}, [f"{ref}: sync waivers must be a list"] + waivers: dict[str, SyncWaiver] = {} + invalid: list[str] = [] + for row in rows: + try: + waiver = _waiver(row) + except ValueError as exc: + invalid.append(f"{ref}: {exc}") + continue + if waiver.waiver_id in waivers: + invalid.append(f"{ref}: duplicate waiver id {waiver.waiver_id}") + continue + waivers[waiver.waiver_id] = waiver + return waivers, invalid + + +def load_sync_waivers( + root: Path, + manifest: UnitManifest, + *, + now: datetime, +) -> WaiverSet: + """Load an immutable base/head waiver union without permitting weakening.""" + base, invalid = _load(root, manifest.base_ref) + head, head_invalid = _load(root, manifest.head_ref) + invalid.extend(head_invalid) + effective = dict(base) + for waiver_id, waiver in head.items(): + protected = base.get(waiver_id) + if protected is not None and protected != waiver: + invalid.append(f"candidate changed protected waiver {waiver_id}") + continue + effective[waiver_id] = waiver + invalid.extend( + f"candidate removed protected waiver {waiver_id}" + for waiver_id in sorted(set(base) - set(head)) + ) + expected = {unit.prompt_relpath for unit in manifest.expected_managed} + valid: list[SyncWaiver] = [] + for waiver in effective.values(): + if waiver.prompt_path not in expected: + invalid.append( + f"waiver {waiver.waiver_id} references non-managed subject " + f"{waiver.prompt_path}" + ) + continue + if waiver.expires_at <= now: + invalid.append(f"waiver {waiver.waiver_id} is expired") + continue + valid.append(waiver) + return WaiverSet(tuple(sorted(valid)), tuple(sorted(set(invalid)))) diff --git a/pyproject.toml b/pyproject.toml index 6a527281d..c7b5268e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "Requests==2.34.2", "aiofiles==25.1.0", "click==8.4.1", + "cryptography>=46,<50", "firecrawl-py==4.28.0", "firebase_admin==7.4.0", "httpx==0.28.1", @@ -86,9 +87,11 @@ Issue-Tracker = "https://github.com/promptdriven/pdd/issues" [project.scripts] pdd = "pdd.cli:cli" +pdd-sync-checker = "pdd.sync_core.released_checker:main" +pdd-sync-verify = "pdd.sync_core.certificate_verifier:main" [tool.setuptools] -packages = ["pdd", "pdd.commands", "pdd.core", "pdd.server", "pdd.server.routes"] +packages = ["pdd", "pdd.commands", "pdd.core", "pdd.server", "pdd.server.routes", "pdd.sync_core"] [tool.setuptools.package-data] "pdd" = [ diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py new file mode 100644 index 000000000..08c27418d --- /dev/null +++ b/tests/test_sync_core_certificate.py @@ -0,0 +1,481 @@ +"""Tests for the signed cross-repository certificate predicate.""" + +import json +import subprocess +from datetime import datetime, timedelta, timezone + +import pytest + +from pdd.sync_core import ( + AttestationSigner, + CheckerIdentity, + GlobalCertificateOptions, + LifecycleResult, + RepositoryTarget, + build_global_certificate, + count_vendored_sync_semantics, + verify_global_certificate, +) +from pdd.sync_core.lifecycle import _isolated_lifecycle_environment +from pdd.sync_core.certificate import _nightly_lineage + + +CHECKER = CheckerIdentity( + "c" * 64, + "refs/tags/v-test-checker", + "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v-test-checker", +) + + +@pytest.fixture(autouse=True) +def _avoid_real_clones(monkeypatch): + monkeypatch.setattr( + "pdd.sync_core.certificate._certification_targets", + lambda targets, _directory: targets, + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._nightly_lineage", lambda _row, _targets: True + ) + + +def _report(name): + return { + "ok": True, + "project_root": name, + "repository_id": f"repository-{name}", + "base_sha": "a" * 40, + "head_sha": "b" * 40, + "errors": [], + "recovery_required": [], + "counts": { + "unaccounted_tracked_paths": 0, + "managed_units": 1, + "protected_expected_managed_units": 1, + "managed_waivers": 0, + "trusted_in_sync": 1, + "verification_profile_complete": 1, + "trusted_current_evidence": 1, + "drifted": 0, + "unbaselined": 0, + "corrupt": 0, + "unknown": 0, + "conflict": 0, + "failed": 0, + "invalid": 0, + }, + } + + +def _nightly(path, signer, count=7): + start = datetime(2026, 7, 3, tzinfo=timezone.utc) + rows = [] + for index in range(count): + reports = [ + {**_report("pdd"), "repository": "pdd"}, + {**_report("pdd_cloud"), "repository": "pdd_cloud"}, + ] + counts = { + "unaccounted_tracked_paths": 0, + "managed_units": 2, + "protected_expected_managed_units": 2, + "managed_waivers": 0, + "trusted_in_sync": 2, + "verification_profile_complete": 2, + "trusted_current_evidence": 2, + "DRIFTED": 0, + "UNBASELINED": 0, + "CORRUPT": 0, + "UNKNOWN": 0, + "CONFLICT": 0, + "FAILED": 0, + "INVALID": 0, + "pdd_cloud_vendored_sync_semantics": 0, + "nightly_streak": index, + "required_nightly_streak": 7, + "verification_profile_coverage": 100, + "trusted_current_evidence_coverage": 100, + } + lifecycle = { + "lifecycle_matrix_failed": 0, + "required_tests_skipped_or_xfailed": 0, + "collection_errors": 0, + "timeouts": 0, + "post_repair_second_run_writes": 0, + "post_merge_tree_changes": 0, + "missing_scenarios": [], + } + body = { + "schema_version": 2, + "checked_at": (start + timedelta(days=index)).isoformat(), + "checker": CHECKER.payload(), + "repositories": reports, + "counts": counts, + "lifecycle": lifecycle, + "scan_ok": True, + "ok": index >= 7, + } + rows.append( + { + **body, + "signature": { + "algorithm": "Ed25519", + "issuer": signer.issuer, + "value": signer.sign_bytes( + json.dumps( + body, sort_keys=True, separators=(",", ":") + ).encode() + ), + }, + } + ) + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def _expected_ids(certificate): + return { + report["repository"]: report["repository_id"] + for report in certificate["repositories"] + } + + +def test_lifecycle_child_environment_excludes_all_signing_material( + tmp_path, monkeypatch +) -> None: + monkeypatch.setenv("PDD_CERTIFICATE_SIGNING_KEY", "certificate-secret") + monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "attestation-secret") + monkeypatch.setenv("OPENAI_API_KEY", "provider-secret") + environment = _isolated_lifecycle_environment(tmp_path / "isolated-home") + assert "PDD_CERTIFICATE_SIGNING_KEY" not in environment + assert "PDD_ATTESTATION_SIGNING_KEY" not in environment + assert "OPENAI_API_KEY" not in environment + assert environment["HOME"] == str(tmp_path / "isolated-home") + + +def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatch) -> None: + pdd = tmp_path / "pdd" + cloud = tmp_path / "pdd_cloud" + pdd.mkdir() + cloud.mkdir() + nightly = tmp_path / "nightly.jsonl" + signer = AttestationSigner("certificate-ci", b"g" * 32) + _nightly(nightly, signer) + monkeypatch.setattr( + "pdd.sync_core.certificate.build_canonical_report", + lambda root, _options: _report(str(root)), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._validate_target", lambda target: target + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 + ) + lifecycle = LifecycleResult(0, 0, 0, 0, 0, 0) + certificate = build_global_certificate( + ( + RepositoryTarget("pdd", pdd), + RepositoryTarget("pdd_cloud", cloud), + ), + GlobalCertificateOptions( + tmp_path / "replay", lifecycle, nightly, checker_identity=CHECKER + ), + signer=signer, + ) + assert certificate["ok"] is True + assert certificate["counts"]["managed_units"] == 2 + assert certificate["counts"]["verification_profile_coverage"] == 100 + assert certificate["counts"]["trusted_current_evidence_coverage"] == 100 + expected = { + "pdd": ("a" * 40, "b" * 40), + "pdd_cloud": ("a" * 40, "b" * 40), + } + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + ) is True + certificate["counts"]["managed_units"] = 0 + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + ) is False + + +def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> None: + for name in ("pdd", "pdd_cloud"): + (tmp_path / name).mkdir() + nightly = tmp_path / "nightly.jsonl" + signer = AttestationSigner("certificate-ci", b"g" * 32) + _nightly(nightly, signer) + monkeypatch.setattr( + "pdd.sync_core.certificate.build_canonical_report", + lambda root, _options: _report(str(root)), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._validate_target", lambda target: target + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 + ) + lifecycle = LifecycleResult(0, 0, 0, 0, 0, 0, ("merge-group",)) + certificate = build_global_certificate( + ( + RepositoryTarget("pdd", tmp_path / "pdd"), + RepositoryTarget("pdd_cloud", tmp_path / "pdd_cloud"), + ), + GlobalCertificateOptions( + tmp_path / "replay", lifecycle, nightly, checker_identity=CHECKER + ), + signer=signer, + ) + assert certificate["ok"] is False + expected = { + "pdd": ("a" * 40, "b" * 40), + "pdd_cloud": ("a" * 40, "b" * 40), + } + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + ) is False + + +def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( + tmp_path, monkeypatch +) -> None: + for name in ("pdd", "pdd_cloud"): + (tmp_path / name).mkdir() + signer = AttestationSigner("certificate-ci", b"g" * 32) + nightly = tmp_path / "nightly.jsonl" + _nightly(nightly, signer) + monkeypatch.setattr( + "pdd.sync_core.certificate.build_canonical_report", + lambda root, _options: _report(str(root)), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._validate_target", lambda target: target + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.count_vendored_sync_semantics", + lambda *_args, **_kwargs: 0, + ) + certificate = build_global_certificate( + ( + RepositoryTarget("pdd", tmp_path / "pdd"), + RepositoryTarget("pdd_cloud", tmp_path / "pdd_cloud"), + ), + GlobalCertificateOptions( + tmp_path / "replay", + LifecycleResult(0, 0, 0, 0, 0, 0), + nightly, + checker_identity=CHECKER, + ), + signer=signer, + ) + expected = { + "pdd": ("a" * 40, "b" * 40), + "pdd_cloud": ("a" * 40, "b" * 40), + } + checked = datetime.fromisoformat(certificate["checked_at"]) + common = { + "expected_issuer": signer.issuer, + "expected_repository_shas": expected, + "expected_repository_ids": _expected_ids(certificate), + "expected_checker_identity": CHECKER, + } + assert verify_global_certificate( + certificate, signer.public_key_bytes(), **common + ) is True + other_checker = CheckerIdentity( + "d" * 64, + CHECKER.release_ref, + CHECKER.workflow_identity, + ) + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=other_checker, + ) is False + wrong_ids = {**_expected_ids(certificate), "pdd_cloud": "wrong-repository"} + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=wrong_ids, + expected_checker_identity=CHECKER, + ) is False + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + now=checked + timedelta(hours=1), + **common, + ) is False + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer="wrong-issuer", + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + ) is False + wrong_refs = {**expected, "pdd_cloud": ("a" * 40, "c" * 40)} + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=wrong_refs, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + ) is False + + +def test_vendored_sync_function_is_counted(tmp_path) -> None: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run( + ["git", "config", "user.email", "certificate@example.com"], + cwd=tmp_path, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "Certificate Test"], + cwd=tmp_path, + check=True, + ) + policy = tmp_path / ".pdd/sync-consumer-boundary.json" + policy.parent.mkdir() + policy.write_text( + json.dumps( + { + "schema_version": 1, + "canonical_package": "pdd.sync_core", + "forbidden_paths": ["**/pdd_fingerprint.py"], + "forbidden_imports": ["pdd.operation_log"], + "forbidden_symbols": ["extract_include_deps"], + "excluded_paths": [], + } + ) + ) + service = tmp_path / "service.py" + service.write_text("def extract_include_deps(path):\n return {}\n") + prompt = tmp_path / "prompts/service_Python.prompt" + prompt.parent.mkdir() + prompt.write_text("Call pdd.operation_log.save_fingerprint after each run.\n") + architecture = tmp_path / "architecture.json" + architecture.write_text( + json.dumps( + [ + { + "filename": "service_Python.prompt", + "description": "Owns extract_include_deps for consumer sync", + } + ] + ) + ) + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "consumer"], cwd=tmp_path, check=True) + assert count_vendored_sync_semantics(tmp_path) == 3 + + +def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( + tmp_path, monkeypatch +) -> None: + for name in ("pdd", "pdd_cloud"): + (tmp_path / name).mkdir() + nightly = tmp_path / "nightly.jsonl" + nightly.write_text( + "".join( + json.dumps({"scan_ok": True, "checked_at": "2026-07-10T00:00:00+00:00"}) + + "\n" + for _ in range(7) + ) + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.build_canonical_report", + lambda root, _options: _report(str(root)), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._validate_target", lambda target: target + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 + ) + certificate = build_global_certificate( + ( + RepositoryTarget("pdd", tmp_path / "pdd"), + RepositoryTarget("pdd_cloud", tmp_path / "pdd_cloud"), + ), + GlobalCertificateOptions( + tmp_path / "replay", + LifecycleResult(0, 0, 0, 0, 0, 0), + nightly, + checker_identity=CHECKER, + ), + signer=AttestationSigner("certificate-ci", b"g" * 32), + ) + assert certificate["counts"]["nightly_streak"] == 0 + assert certificate["scan_ok"] is True + assert certificate["ok"] is False + + +def test_nightly_lineage_requires_identity_and_ancestry(tmp_path) -> None: + targets = [] + reports = [] + for name in ("pdd", "pdd_cloud"): + root = tmp_path / name + root.mkdir() + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + subprocess.run( + ["git", "config", "user.email", "lineage@example.com"], + cwd=root, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "Lineage Test"], cwd=root, check=True + ) + (root / ".pdd").mkdir() + identity = f"identity-{name}" + (root / ".pdd/repository-id").write_text(identity + "\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=root, check=True) + historical = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + (root / "later").write_text("current\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-qm", "head"], cwd=root, check=True) + current = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + targets.append(RepositoryTarget(name, root, historical, current)) + reports.append( + { + "repository": name, + "repository_id": identity, + "head_sha": historical, + } + ) + row = {"repositories": reports} + assert _nightly_lineage(row, tuple(targets)) is True + reports[1]["repository_id"] = "wrong-identity" + assert _nightly_lineage(row, tuple(targets)) is False diff --git a/tests/test_sync_core_certificate_verifier.py b/tests/test_sync_core_certificate_verifier.py new file mode 100644 index 000000000..a32573253 --- /dev/null +++ b/tests/test_sync_core_certificate_verifier.py @@ -0,0 +1,63 @@ +"""Tests for protected independent certificate expectations.""" + +import base64 +import json + +import pytest + +from pdd.sync_core.certificate_verifier import load_expectations + + +def _payload(): + return { + "schema_version": 1, + "issuer": "global-sync-checker", + "public_key_base64": base64.b64encode(b"k" * 32).decode(), + "checker": { + "wheel_sha256": "c" * 64, + "release_ref": "refs/tags/v1.0.0", + "workflow_identity": ( + "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v1.0.0" + ), + }, + "repositories": { + "pdd": { + "repository_id": "pdd-id", + "base_sha": "a" * 40, + "head_sha": "b" * 40, + }, + "pdd_cloud": { + "repository_id": "cloud-id", + "base_sha": "c" * 40, + "head_sha": "d" * 40, + }, + }, + } + + +def test_expectations_bind_ids_shas_checker_and_key(tmp_path) -> None: + path = tmp_path / "expectations.json" + path.write_text(json.dumps(_payload())) + expected = load_expectations(path) + assert expected.repository_ids == {"pdd": "pdd-id", "pdd_cloud": "cloud-id"} + assert expected.repository_shas["pdd"] == ("a" * 40, "b" * 40) + assert expected.checker.wheel_sha256 == "c" * 64 + assert expected.public_key == b"k" * 32 + + +@pytest.mark.parametrize( + "mutation", + [ + lambda payload: payload["repositories"].pop("pdd_cloud"), + lambda payload: payload["repositories"]["pdd"].update(base_sha="short"), + lambda payload: payload["repositories"]["pdd"].update(repository_id=""), + lambda payload: payload.update(public_key_base64="not-base64"), + ], +) +def test_malformed_expectations_fail_closed(tmp_path, mutation) -> None: + payload = _payload() + mutation(payload) + path = tmp_path / "expectations.json" + path.write_text(json.dumps(payload)) + with pytest.raises(ValueError): + load_expectations(path) diff --git a/tests/test_sync_core_classifier.py b/tests/test_sync_core_classifier.py new file mode 100644 index 000000000..a9f4ab098 --- /dev/null +++ b/tests/test_sync_core_classifier.py @@ -0,0 +1,321 @@ +"""Contract tests for the side-effect-free canonical sync classifier.""" + +from datetime import datetime, timezone +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + ArtifactSnapshot, + AttestationBinding, + AttestationRequest, + AttestationSigner, + AttestationTrustPolicy, + BaselineStatus, + EvidenceOutcome, + FingerprintRecord, + FingerprintProvenance, + InventoryStatus, + SemanticStatus, + UnitId, + UnitSnapshot, + ValidationEvidence, + VerificationObligation, + VerificationProfile, + classify, +) +from pdd.sync_core.types import ObligationEvidence + + +UNIT_ID = UnitId("repository-1", PurePosixPath("prompts/widget_python.prompt"), "python") +NOW = datetime(2026, 7, 10, 12, 0, tzinfo=timezone.utc) +PRIVATE_KEY = b"c" * 32 +SIGNER = AttestationSigner("trusted-ci", PRIVATE_KEY) +PUBLIC_KEY = SIGNER.public_key_bytes() + + +def _snapshot(**digests: str | None) -> UnitSnapshot: + values = { + "prompt": "prompt-v1", + "include": "include-v1", + "code": "code-v1", + "example": "example-v1", + "test": "test-v1", + } + values.update(digests) + return UnitSnapshot( + UNIT_ID, + tuple( + ArtifactSnapshot( + role, + PurePosixPath( + { + "prompt": "prompts/widget_python.prompt", + "include": "docs/widget.md", + "code": "src/widget.py", + "example": "examples/widget_example.py", + "test": "tests/test_widget.py", + }[role] + ), + digest, + "100644" if digest is not None else None, + ) + for role, digest in values.items() + ), + manifest_digest="manifest-v1", + dependency_snapshot_digest="graph-v1", + verification_profile_digest="profile-v1", + ) + + +def _profile() -> VerificationProfile: + return VerificationProfile( + UNIT_ID, + ( + VerificationObligation( + "tests", + "test", + "pytest", + "pytest-config-v1", + ("REQ-1",), + (PurePosixPath("tests/test_widget.py"),), + ), + ), + ("REQ-1",), + "profile-v1", + ) + + +def _evidence(snapshot: UnitSnapshot, outcome: EvidenceOutcome = EvidenceOutcome.PASS) -> ValidationEvidence: + binding = AttestationBinding( + UNIT_ID, + snapshot.digest(), + "profile-v1", + "pytest-v1", + "pdd-test", + "base-1", + "head-1", + ) + envelope = SIGNER.issue( + AttestationRequest( + "attestation-1", + binding, + (ObligationEvidence("tests", outcome),), + "nonce-1", + NOW, + ) + ) + return AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}).verify( + envelope, binding, now=NOW + ) + + +def _baseline(snapshot: UnitSnapshot) -> FingerprintRecord: + provenance = FingerprintProvenance( + "generated", + "pdd sync widget", + "transaction-1", + "head-1", + "2026-07-10T12:00:00+00:00", + "pdd-test", + ) + return FingerprintRecord( + snapshot, 2, 2, provenance, SemanticStatus.VERIFIED, "attestation-1" + ) + + +def test_current_hashes_without_evidence_are_not_in_sync() -> None: + snapshot = _snapshot() + verdict = classify(snapshot, _baseline(snapshot), _profile(), None) + assert verdict.baseline is BaselineStatus.CURRENT + assert verdict.semantic is SemanticStatus.UNKNOWN + assert verdict.in_sync is False + + +def test_complete_trusted_current_evidence_is_in_sync() -> None: + snapshot = _snapshot() + verdict = classify(snapshot, _baseline(snapshot), _profile(), _evidence(snapshot)) + assert verdict.semantic is SemanticStatus.VERIFIED + assert verdict.in_sync is True + + +@pytest.mark.parametrize("outcome", [EvidenceOutcome.SKIP, EvidenceOutcome.XFAIL, EvidenceOutcome.FLAKY]) +def test_non_passing_evidence_cannot_verify(outcome) -> None: + snapshot = _snapshot() + verdict = classify(snapshot, _baseline(snapshot), _profile(), _evidence(snapshot, outcome)) + assert verdict.semantic is SemanticStatus.UNKNOWN + assert verdict.in_sync is False + + +def test_prompt_and_code_coedit_is_conflict() -> None: + baseline = _snapshot() + current = _snapshot(prompt="prompt-v2", code="code-v2") + verdict = classify(current, _baseline(baseline), _profile(), _evidence(current)) + assert verdict.baseline is BaselineStatus.DRIFTED + assert verdict.semantic is SemanticStatus.CONFLICT + assert verdict.changed_roles == ("code", "prompt") + assert verdict.in_sync is False + + +@pytest.mark.parametrize("role", ["prompt", "include", "code", "example", "test"]) +def test_one_sided_drift_never_accepts_current_hashes(role: str) -> None: + baseline = _snapshot() + current = _snapshot(**{role: f"{role}-v2"}) + verdict = classify(current, _baseline(baseline), _profile(), _evidence(current)) + assert verdict.baseline is BaselineStatus.DRIFTED + assert verdict.semantic is SemanticStatus.UNKNOWN + assert verdict.in_sync is False + + +def test_missing_required_artifact_fails() -> None: + baseline = _snapshot() + current = _snapshot(code=None) + verdict = classify(current, _baseline(baseline), _profile(), None) + assert verdict.baseline is BaselineStatus.DRIFTED + assert verdict.semantic is SemanticStatus.FAILED + assert verdict.required_artifacts_missing == ("code",) + + +def test_mode_only_change_is_drift() -> None: + baseline = _snapshot() + artifacts = tuple( + ArtifactSnapshot( + item.role, + item.relpath, + item.digest, + "100755" if item.role == "code" else item.git_mode, + ) + for item in baseline.artifacts + ) + current = UnitSnapshot( + UNIT_ID, + artifacts, + baseline.manifest_digest, + baseline.dependency_snapshot_digest, + baseline.verification_profile_digest, + ) + verdict = classify(current, _baseline(baseline), _profile(), None) + assert verdict.baseline is BaselineStatus.DRIFTED + assert verdict.changed_roles == ("code",) + + +def test_missing_baseline_is_unknown_not_current() -> None: + verdict = classify(_snapshot(), None, _profile(), None) + assert verdict.baseline is BaselineStatus.UNBASELINED + assert verdict.semantic is SemanticStatus.UNKNOWN + assert verdict.in_sync is False + + +def test_invalid_inventory_fails_closed() -> None: + snapshot = _snapshot() + verdict = classify( + snapshot, + _baseline(snapshot), + _profile(), + _evidence(snapshot), + inventory=InventoryStatus.INVALID, + ) + assert verdict.baseline is BaselineStatus.CORRUPT + assert verdict.semantic is SemanticStatus.FAILED + assert verdict.in_sync is False + + +def test_empty_profile_and_evidence_cannot_verify() -> None: + snapshot = _snapshot() + profile = VerificationProfile(UNIT_ID, (), (), "profile-v1") + binding = AttestationBinding( + UNIT_ID, + snapshot.digest(), + "profile-v1", + "pytest-v1", + "pdd-test", + "base-1", + "head-1", + ) + envelope = SIGNER.issue( + AttestationRequest( + "attestation-empty", + binding, + (), + "nonce-empty", + NOW, + ) + ) + evidence = AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}).verify( + envelope, binding, now=NOW + ) + verdict = classify(snapshot, _baseline(snapshot), profile, evidence) + assert profile.complete is False + assert verdict.semantic is SemanticStatus.UNKNOWN + assert verdict.in_sync is False + + +def test_duplicate_obligation_ids_are_rejected() -> None: + obligation = VerificationObligation( + "duplicate", + "test", + "pytest", + "pytest-v1", + ("REQ-1",), + (PurePosixPath("tests/test_widget.py"),), + ) + with pytest.raises(ValueError, match="duplicate obligation"): + VerificationProfile( + UNIT_ID, + (obligation, obligation), + ("REQ-1",), + "profile-duplicate", + ) + + +def test_multiple_artifacts_with_same_role_are_preserved() -> None: + snapshot = _snapshot() + second_test = ArtifactSnapshot( + "test", + PurePosixPath("tests/test_widget_integration.py"), + "integration-test-v1", + "100644", + ) + baseline = UnitSnapshot( + snapshot.unit_id, + snapshot.artifacts + (second_test,), + snapshot.manifest_digest, + snapshot.dependency_snapshot_digest, + snapshot.verification_profile_digest, + ) + changed_test = ArtifactSnapshot( + "test", + second_test.relpath, + "integration-test-v2", + "100644", + ) + current = UnitSnapshot( + snapshot.unit_id, + snapshot.artifacts + (changed_test,), + snapshot.manifest_digest, + snapshot.dependency_snapshot_digest, + snapshot.verification_profile_digest, + ) + verdict = classify(current, _baseline(baseline), _profile(), None) + assert verdict.changed_roles == ("test",) + assert len(current.artifact_map()) == 6 + + +def test_nondeterministic_input_cannot_be_verified_by_execution_evidence() -> None: + snapshot = _snapshot() + nondeterministic = UnitSnapshot( + snapshot.unit_id, + snapshot.artifacts, + snapshot.manifest_digest, + snapshot.dependency_snapshot_digest, + snapshot.verification_profile_digest, + nondeterministic_inputs=True, + ) + verdict = classify( + nondeterministic, + _baseline(nondeterministic), + _profile(), + _evidence(nondeterministic), + ) + assert verdict.semantic is SemanticStatus.UNKNOWN + assert verdict.in_sync is False diff --git a/tests/test_sync_core_cli.py b/tests/test_sync_core_cli.py new file mode 100644 index 000000000..9448b0683 --- /dev/null +++ b/tests/test_sync_core_cli.py @@ -0,0 +1,43 @@ +"""CLI contract tests for canonical synchronization certification.""" + +from click.testing import CliRunner + +from pdd.cli import cli + + +def test_documented_sync_certify_spelling_is_registered() -> None: + result = CliRunner().invoke(cli, ["sync", "certify", "--help"]) + assert result.exit_code == 0, result.output + assert "--repos" in result.output + assert "--run-lifecycle-matrix" in result.output + + +def test_global_certify_requires_complete_acceptance_inputs(tmp_path) -> None: + replay = tmp_path / "replay" + result = CliRunner().invoke( + cli, + [ + "sync", + "certify", + "--repos", + "pdd,pdd_cloud", + "--replay-ledger", + str(replay), + ], + ) + assert result.exit_code == 1 + assert "global certification requires" in result.output + assert "--full-inventory" in result.output + + +def test_reviewed_baseline_command_is_registered() -> None: + result = CliRunner().invoke(cli, ["baseline", "--help"]) + assert result.exit_code == 0, result.output + assert "--reviewed-by" in result.output + assert "--reason" in result.output + + +def test_trusted_validate_command_is_registered() -> None: + result = CliRunner().invoke(cli, ["validate", "--help"]) + assert result.exit_code == 0, result.output + assert "--base-ref" in result.output diff --git a/tests/test_sync_core_evidence_store.py b/tests/test_sync_core_evidence_store.py new file mode 100644 index 000000000..fe71cd42a --- /dev/null +++ b/tests/test_sync_core_evidence_store.py @@ -0,0 +1,101 @@ +"""Tests for untrusted evidence storage and protected-base trust roots.""" + +import base64 +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + +from pdd.sync_core import ( + AttestationBinding, + AttestationRequest, + AttestationSigner, + EvidenceOutcome, + decode_attestation, + encode_attestation, + load_trust_policy, +) +from pdd.sync_core.types import ObligationEvidence, UnitId + + +PRIVATE_KEY = b"e" * 32 +SIGNER = AttestationSigner("trusted-ci", PRIVATE_KEY) +UNIT = UnitId("repository-1", PurePosixPath("prompts/widget_python.prompt"), "python") + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _envelope(): + binding = AttestationBinding( + UNIT, "snapshot", "profile", "runner", "pdd-test", "base", "head" + ) + return SIGNER.issue( + AttestationRequest( + "attestation-1", + binding, + (ObligationEvidence("test", EvidenceOutcome.PASS),), + "nonce-1", + datetime(2026, 7, 10, 12, 0, tzinfo=timezone.utc), + ) + ) + + +def test_attestation_json_round_trip_preserves_signed_payload() -> None: + envelope = _envelope() + decoded = decode_attestation(json.loads(encode_attestation(envelope))) + assert decoded == envelope + assert decoded.payload() == envelope.payload() + + +def test_candidate_cannot_replace_protected_base_public_key(tmp_path) -> None: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "trust@example.com") + _git(root, "config", "user.name", "Trust Test") + policy_path = root / ".pdd/attestation-trust.json" + policy_path.parent.mkdir() + protected_key = base64.b64encode(SIGNER.public_key_bytes()).decode("ascii") + policy_path.write_text( + json.dumps( + { + "issuers": [ + {"issuer_id": "trusted-ci", "public_key": protected_key} + ], + "revoked_issuers": [], + "revoked_attestations": [], + } + ) + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "protected trust") + base = _git(root, "rev-parse", "HEAD") + candidate = AttestationSigner("trusted-ci", b"x" * 32) + policy_path.write_text( + json.dumps( + { + "issuers": [ + { + "issuer_id": "trusted-ci", + "public_key": base64.b64encode( + candidate.public_key_bytes() + ).decode("ascii"), + } + ] + } + ) + ) + loaded = load_trust_policy( + root, + base, + replay_ledger_path=tmp_path / "external-trust/replay.json", + ) + assert loaded.issuer_ids == ("trusted-ci",) + assert loaded.digest + assert loaded.verifier.verify( + _envelope(), _envelope().binding, now=datetime(2026, 7, 10, 12, 1, tzinfo=timezone.utc) + ) diff --git a/tests/test_sync_core_fingerprint_store.py b/tests/test_sync_core_fingerprint_store.py new file mode 100644 index 000000000..9c5458957 --- /dev/null +++ b/tests/test_sync_core_fingerprint_store.py @@ -0,0 +1,168 @@ +"""Tests for canonical v2 fingerprint persistence and migration safety.""" + +import json +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + ArtifactSnapshot, + CorruptFingerprintError, + FingerprintProvenance, + FingerprintRecord, + FingerprintStore, + FingerprintStoreError, + SemanticStatus, + UnitId, + UnitSnapshot, +) +from pdd.sync_core.identity import initialize_repository_identity + + +REPOSITORY_ID = "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" +UNIT_ID = UnitId( + REPOSITORY_ID, PurePosixPath("prompts/widget_python.prompt"), "python" +) + + +def _store(tmp_path): + initialize_repository_identity(tmp_path, REPOSITORY_ID) + return FingerprintStore(tmp_path) + + +def _record(*, kind="generated", semantic=SemanticStatus.VERIFIED, attestation="att-1"): + snapshot = UnitSnapshot( + UNIT_ID, + ( + ArtifactSnapshot( + "prompt", + PurePosixPath("prompts/widget_python.prompt"), + "prompt-hash", + "100644", + ), + ArtifactSnapshot( + "test", + PurePosixPath("tests/test_widget.py"), + "test-hash", + "100644", + ), + ArtifactSnapshot( + "test", + PurePosixPath("tests/test_widget_e2e.py"), + "e2e-hash", + "100755", + ), + ), + "manifest-hash", + "graph-hash", + "profile-hash", + ) + provenance = FingerprintProvenance( + kind, + "pdd sync widget", + "transaction-1", + "head-1", + "2026-07-10T12:00:00+00:00", + "pdd-test", + "reviewer@example.com" if kind == "baseline-reset" else None, + "reviewed migration" if kind == "baseline-reset" else None, + ) + return FingerprintRecord(snapshot, 2, 2, provenance, semantic, attestation) + + +def test_v2_round_trip_preserves_all_artifact_paths_and_modes(tmp_path) -> None: + store = _store(tmp_path) + record = _record() + path = store.write(record) + assert path.stat().st_mode & 0o777 == 0o644 + assert store.load(UNIT_ID) == record + assert len(store.load(UNIT_ID).snapshot.artifacts) == 3 + + +def test_corrupt_existing_record_fails_without_rewriting(tmp_path) -> None: + store = _store(tmp_path) + path = store.write(_record()) + path.write_text("{not-json", encoding="utf-8") + before = path.read_bytes() + with pytest.raises(CorruptFingerprintError): + store.load(UNIT_ID) + assert path.read_bytes() == before + + +def test_required_null_hash_is_rejected(tmp_path) -> None: + store = _store(tmp_path) + record = _record() + missing = ArtifactSnapshot( + "code", PurePosixPath("src/widget.py"), None, None, required=True + ) + snapshot = UnitSnapshot( + record.snapshot.unit_id, + record.snapshot.artifacts + (missing,), + record.snapshot.manifest_digest, + record.snapshot.dependency_snapshot_digest, + record.snapshot.verification_profile_digest, + ) + invalid = FingerprintRecord( + snapshot, + 2, + 2, + record.provenance, + record.claimed_semantic_status, + record.attestation_ref, + ) + with pytest.raises(FingerprintStoreError, match="null hash or mode"): + store.write(invalid) + + +def test_verified_record_requires_attestation(tmp_path) -> None: + with pytest.raises(FingerprintStoreError, match="requires an attestation"): + _store(tmp_path).write(_record(attestation=None)) + + +def test_baseline_reset_is_current_but_semantically_unknown(tmp_path) -> None: + store = _store(tmp_path) + record = _record( + kind="baseline-reset", semantic=SemanticStatus.UNKNOWN, attestation=None + ) + store.write(record) + assert store.load(UNIT_ID).claimed_semantic_status is SemanticStatus.UNKNOWN + with pytest.raises(FingerprintStoreError, match="must remain semantic UNKNOWN"): + store.write(_record(kind="baseline-reset")) + + +def test_baseline_reset_requires_review_audit_fields(tmp_path) -> None: + record = _record( + kind="baseline-reset", semantic=SemanticStatus.UNKNOWN, attestation=None + ) + provenance = FingerprintProvenance( + record.provenance.kind, + record.provenance.command, + record.provenance.transaction_id, + record.provenance.git_commit, + record.provenance.timestamp, + record.provenance.pdd_version, + ) + with pytest.raises(FingerprintStoreError, match="reviewer and rationale"): + _store(tmp_path).write( + FingerprintRecord(record.snapshot, 2, 2, provenance, SemanticStatus.UNKNOWN, None) + ) + + +def test_legacy_record_is_readable_but_not_promoted(tmp_path) -> None: + store = _store(tmp_path) + legacy_path = tmp_path / ".pdd/meta/widget_python.json" + legacy_path.parent.mkdir(parents=True, exist_ok=True) + legacy_path.write_text(json.dumps({"prompt_hash": "legacy"}), encoding="utf-8") + legacy = store.read_legacy(legacy_path) + assert legacy.payload["prompt_hash"] == "legacy" + assert store.load(UNIT_ID) is None + + +def test_embedded_identity_mismatch_is_corrupt(tmp_path) -> None: + store = _store(tmp_path) + path = store.write(_record()) + payload = json.loads(path.read_text(encoding="utf-8")) + payload["unit_id"]["prompt_relpath"] = "prompts/other_python.prompt" + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(CorruptFingerprintError, match="embedded identity"): + store.load(UNIT_ID) diff --git a/tests/test_sync_core_identity_path_policy.py b/tests/test_sync_core_identity_path_policy.py new file mode 100644 index 000000000..277307506 --- /dev/null +++ b/tests/test_sync_core_identity_path_policy.py @@ -0,0 +1,114 @@ +"""Tests for stable repository identity and protected path handling.""" + +import os +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + PathPolicy, + PathPolicyError, + RepositoryIdentityError, + initialize_repository_identity, + read_repository_identity, +) + + +def test_repository_identity_is_initialized_once(tmp_path) -> None: + expected = "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + identity = initialize_repository_identity(tmp_path, expected) + assert identity.value == expected + assert identity.persistent is True + assert read_repository_identity(tmp_path) == identity + assert initialize_repository_identity(tmp_path, "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa") == identity + + +def test_legacy_identity_is_report_only(tmp_path) -> None: + first = read_repository_identity(tmp_path) + second = read_repository_identity(tmp_path) + assert first == second + assert first.persistent is False + with pytest.raises(RepositoryIdentityError, match="requires initialization"): + read_repository_identity(tmp_path, require_persistent=True) + + +def test_malformed_repository_identity_fails_closed(tmp_path) -> None: + identity_path = tmp_path / ".pdd/repository-id" + identity_path.parent.mkdir() + identity_path.write_text("candidate-controlled-name\n", encoding="ascii") + with pytest.raises(RepositoryIdentityError, match="valid UUID"): + read_repository_identity(tmp_path) + + +@pytest.fixture() +def repository(tmp_path): + (tmp_path / "src").mkdir() + (tmp_path / "src/widget.py").write_text("print('widget')\n", encoding="utf-8") + return tmp_path + + +def test_snapshot_binds_content_and_executable_mode(repository) -> None: + policy = PathPolicy(repository) + relpath = PurePosixPath("src/widget.py") + regular = policy.snapshot("code", relpath) + os.chmod(repository / relpath, 0o755) + executable = policy.snapshot("code", relpath) + assert regular.digest == executable.digest + assert regular.git_mode == "100644" + assert executable.git_mode == "100755" + + +@pytest.mark.parametrize( + "relpath", + [PurePosixPath("../outside.py"), PurePosixPath("/tmp/outside.py")], +) +def test_path_policy_rejects_lexical_escape(repository, relpath) -> None: + with pytest.raises(PathPolicyError, match="repository-relative"): + PathPolicy(repository).resolve(relpath) + + +def test_path_policy_rejects_unapproved_symlink(repository, tmp_path) -> None: + outside = tmp_path / "outside.py" + outside.write_text("secret\n", encoding="utf-8") + (repository / "src/alias.py").symlink_to(outside) + with pytest.raises(PathPolicyError, match="unapproved managed symlink"): + PathPolicy(repository).resolve(PurePosixPath("src/alias.py")) + + +def test_path_policy_accepts_unchanged_tracked_alias(repository) -> None: + (repository / "canonical").mkdir() + (repository / "canonical/widget.py").write_text("value = 1\n", encoding="utf-8") + (repository / "alias").symlink_to("canonical", target_is_directory=True) + policy = PathPolicy( + repository, + {PurePosixPath("alias"): PurePosixPath("canonical")}, + ) + resolved = policy.resolve(PurePosixPath("alias/widget.py")) + assert resolved.logical_relpath == PurePosixPath("alias/widget.py") + assert resolved.canonical_path == repository / "canonical/widget.py" + assert resolved.alias_relpath == PurePosixPath("alias") + + +def test_path_policy_rejects_alias_retarget(repository) -> None: + (repository / "canonical").mkdir() + (repository / "canonical/widget.py").write_text("value = 1\n", encoding="utf-8") + (repository / "other").mkdir() + (repository / "other/widget.py").write_text("value = 2\n", encoding="utf-8") + alias = repository / "alias" + alias.symlink_to("canonical", target_is_directory=True) + policy = PathPolicy( + repository, + {PurePosixPath("alias"): PurePosixPath("canonical")}, + ) + policy.resolve(PurePosixPath("alias/widget.py")) + alias.unlink() + alias.symlink_to("other", target_is_directory=True) + with pytest.raises(PathPolicyError, match="target changed"): + policy.resolve(PurePosixPath("alias/widget.py")) + + +def test_path_policy_rejects_special_file(repository) -> None: + fifo = repository / "src/events" + os.mkfifo(fifo) + with pytest.raises(PathPolicyError, match="not a regular file"): + PathPolicy(repository).resolve(PurePosixPath("src/events")) diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py new file mode 100644 index 000000000..9f5183f53 --- /dev/null +++ b/tests/test_sync_core_includes.py @@ -0,0 +1,249 @@ +"""Conformance tests for the parser shared by expansion and sync consumers.""" + +from pdd.preprocess import compute_user_intent_paths +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + IncludeGraphError, + IncludeSyntax, + PathPolicy, + build_include_closure, + parse_include_references, +) +from pdd.sync_order import extract_includes_from_file + + +PROMPT = """ +docs/a.md + +docs/c.md + +docs/d.md, docs/e.md +docs/f.md + +`````` +""" + + +def test_all_include_grammars_produce_typed_ordered_records() -> None: + references = parse_include_references(PROMPT) + assert [item.path for item in references] == [ + "docs/a.md", + "docs/b.md", + "docs/c.md", + "docs/d.md", + "docs/e.md", + "docs/f.md", + "docs/g.md", + ] + assert references[1].select == "Thing" + assert references[1].optional is True + assert references[2].query == "summarize" + assert all(item.expand_dependencies for item in references[3:6]) + assert references[-1].syntax is IncludeSyntax.BACKTICK + + +def test_path_attribute_takes_precedence_over_body() -> None: + reference = parse_include_references( + 'docs/fallback.md' + ) + assert [item.path for item in reference] == ["docs/source.md"] + + +def test_literal_include_tag_in_prose_does_not_consume_later_markup() -> None: + text = ( + "The `` graph is recursive and may mention `` in prose.\n" + "docs/actual.md\n" + ) + references = parse_include_references(text) + assert [item.path for item in references] == ["docs/actual.md"] + + +def test_preprocess_and_sync_order_use_the_same_parser(tmp_path) -> None: + prompt = tmp_path / "widget_python.prompt" + prompt.write_text(PROMPT, encoding="utf-8") + expected = {item.path for item in parse_include_references(PROMPT)} + assert compute_user_intent_paths(PROMPT) == expected + assert extract_includes_from_file(prompt) == expected + + +def test_duplicates_are_preserved_in_typed_records() -> None: + references = parse_include_references( + "docs/a.mddocs/a.md" + ) + assert len(references) == 2 + assert references[0].position < references[1].position + + +def test_transitive_closure_hashes_every_input_and_mode(tmp_path) -> None: + (tmp_path / "prompts").mkdir() + (tmp_path / "docs").mkdir() + (tmp_path / "prompts/widget.prompt").write_text( + "docs/a.md", encoding="utf-8" + ) + (tmp_path / "docs/a.md").write_text( + "docs/b.md", encoding="utf-8" + ) + (tmp_path / "docs/b.md").write_text("leaf\n", encoding="utf-8") + closure = build_include_closure( + PurePosixPath("prompts/widget.prompt"), PathPolicy(tmp_path) + ) + assert [item.relpath.as_posix() for item in closure.artifacts] == [ + "docs/a.md", + "docs/b.md", + ] + assert len(closure.digest()) == 64 + + +def test_include_cycle_is_an_error(tmp_path) -> None: + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts/a.prompt").write_text( + "prompts/b.md", encoding="utf-8" + ) + (tmp_path / "prompts/b.md").write_text( + "prompts/a.prompt", encoding="utf-8" + ) + with pytest.raises(IncludeGraphError, match="cycle"): + build_include_closure(PurePosixPath("prompts/a.prompt"), PathPolicy(tmp_path)) + + +@pytest.mark.parametrize("path", ["/tmp/secret", "../../secret"]) +def test_absolute_and_escaping_include_paths_are_rejected(tmp_path, path) -> None: + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts/a.prompt").write_text( + f"{path}", encoding="utf-8" + ) + with pytest.raises(IncludeGraphError): + build_include_closure(PurePosixPath("prompts/a.prompt"), PathPolicy(tmp_path)) + + +def test_optional_missing_include_is_bound_but_not_failed(tmp_path) -> None: + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts/a.prompt").write_text( + '', encoding="utf-8" + ) + closure = build_include_closure( + PurePosixPath("prompts/a.prompt"), PathPolicy(tmp_path) + ) + assert closure.edges[0].target_exists is False + assert closure.artifacts == () + + +def test_query_include_marks_closure_nondeterministic(tmp_path) -> None: + (tmp_path / "prompts").mkdir() + (tmp_path / "docs").mkdir() + (tmp_path / "docs/a.md").write_text("source\n", encoding="utf-8") + (tmp_path / "prompts/a.prompt").write_text( + 'docs/a.md', encoding="utf-8" + ) + closure = build_include_closure( + PurePosixPath("prompts/a.prompt"), PathPolicy(tmp_path) + ) + assert closure.has_nondeterministic_query is True + + +def test_relative_include_normalizes_only_after_source_join(tmp_path) -> None: + prompt = tmp_path / "prompts/project/pages/widget.prompt" + context = tmp_path / "prompts/project/context/preamble.prompt" + prompt.parent.mkdir(parents=True) + context.parent.mkdir(parents=True) + prompt.write_text("../context/preamble.prompt") + context.write_text("context\n") + closure = build_include_closure( + PurePosixPath("prompts/project/pages/widget.prompt"), PathPolicy(tmp_path) + ) + assert [item.relpath for item in closure.artifacts] == [ + PurePosixPath("prompts/project/context/preamble.prompt") + ] + + +def test_include_searches_ancestor_project_roots_deterministically(tmp_path) -> None: + prompt = tmp_path / "projects/app/prompts/pages/widget.prompt" + source = tmp_path / "projects/app/src/config.py" + prompt.parent.mkdir(parents=True) + source.parent.mkdir(parents=True) + prompt.write_text("src/config.py") + source.write_text("VALUE = 1\n") + closure = build_include_closure( + PurePosixPath("projects/app/prompts/pages/widget.prompt"), PathPolicy(tmp_path) + ) + assert closure.artifacts[0].relpath == PurePosixPath("projects/app/src/config.py") + + +def test_include_many_glob_hashes_every_matching_file(tmp_path) -> None: + prompt = tmp_path / "prompts/widget.prompt" + fixtures = tmp_path / "tests/fixtures" + prompt.parent.mkdir() + fixtures.mkdir(parents=True) + prompt.write_text( + "tests/fixtures/*.yaml", encoding="utf-8" + ) + (fixtures / "a.yaml").write_text("a: 1\n") + (fixtures / "b.yaml").write_text("b: 2\n") + closure = build_include_closure( + PurePosixPath("prompts/widget.prompt"), PathPolicy(tmp_path) + ) + assert [item.relpath.as_posix() for item in closure.artifacts] == [ + "tests/fixtures/a.yaml", + "tests/fixtures/b.yaml", + ] + + +def test_prompt_tree_projects_include_to_generated_project_tree(tmp_path) -> None: + prompt = tmp_path / "prompts/frontend/app/widget.prompt" + config = tmp_path / "frontend/tsconfig.json" + prompt.parent.mkdir(parents=True) + config.parent.mkdir() + prompt.write_text("tsconfig.json") + config.write_text("{}\n") + closure = build_include_closure( + PurePosixPath("prompts/frontend/app/widget.prompt"), PathPolicy(tmp_path) + ) + assert closure.artifacts[0].relpath == PurePosixPath("frontend/tsconfig.json") + + +def test_manifest_output_alias_resolves_generated_code_relative_include(tmp_path) -> None: + prompt = tmp_path / "prompts/frontend/components/widget.prompt" + dependency = tmp_path / "frontend/src/context/AuthContext.tsx" + prompt.parent.mkdir(parents=True) + dependency.parent.mkdir(parents=True) + prompt.write_text("../context/AuthContext.tsx") + dependency.write_text("export const AuthContext = {};\n") + closure = build_include_closure( + PurePosixPath("prompts/frontend/components/widget.prompt"), + PathPolicy(tmp_path), + root_aliases=(PurePosixPath("frontend/src/components/widget.tsx"),), + ) + assert closure.artifacts[0].relpath == PurePosixPath( + "frontend/src/context/AuthContext.tsx" + ) + + +def test_parent_prefixed_repository_path_stays_inside_checkout(tmp_path) -> None: + prompt = tmp_path / "prompts/frontend/components/widget.prompt" + dependency = tmp_path / "frontend/src/context/AuthContext.tsx" + prompt.parent.mkdir(parents=True) + dependency.parent.mkdir(parents=True) + prompt.write_text("../../frontend/src/context/AuthContext.tsx") + dependency.write_text("export const AuthContext = {};\n") + closure = build_include_closure( + PurePosixPath("prompts/frontend/components/widget.prompt"), + PathPolicy(tmp_path), + ) + assert closure.artifacts[0].relpath == PurePosixPath( + "frontend/src/context/AuthContext.tsx" + ) + + +def test_unresolved_external_package_forces_nondeterministic_closure(tmp_path) -> None: + prompt = tmp_path / "prompts/widget.prompt" + prompt.parent.mkdir() + prompt.write_text("next/navigation") + closure = build_include_closure( + PurePosixPath("prompts/widget.prompt"), PathPolicy(tmp_path) + ) + assert closure.artifacts == () + assert closure.edges[0].target_exists is False + assert closure.has_nondeterministic_query is True diff --git a/tests/test_sync_core_language.py b/tests/test_sync_core_language.py new file mode 100644 index 000000000..6bbe301ab --- /dev/null +++ b/tests/test_sync_core_language.py @@ -0,0 +1,45 @@ +"""Exhaustive conformance tests for every bundled language registry row.""" + +import csv +from pathlib import Path + +import pytest + +from pdd.sync_core import LanguageRegistry, LanguageRegistryError + + +CSV_PATH = Path(__file__).parents[1] / "pdd/data/language_format.csv" + + +def _rows(): + with CSV_PATH.open(newline="", encoding="utf-8") as handle: + return tuple(csv.DictReader(handle)) + + +@pytest.mark.parametrize("row", _rows(), ids=lambda row: row["language"] + row["extension"]) +def test_every_bundled_row_resolves_by_explicit_language(row) -> None: + registry = LanguageRegistry.bundled() + spec = registry.resolve_alias(row["language"]) + extension = row["extension"].casefold() + assert extension in spec.extensions + if extension: + assert registry.resolve_extension( + extension, explicit_language=row["language"] + ) == spec + declared_roles = {role for role in row["outputs"].split("|") if role} + assert declared_roles.issubset(spec.output_roles) + + +@pytest.mark.parametrize("extension", [".sh", ".pl", ".m", ".prompt"]) +def test_ambiguous_extension_never_selects_first_row(extension) -> None: + with pytest.raises(LanguageRegistryError, match="ambiguous or unknown"): + LanguageRegistry.bundled().resolve_extension(extension) + + +def test_duplicate_language_rows_merge_all_extensions() -> None: + yaml = LanguageRegistry.bundled().resolve_alias("YAML") + assert yaml.extensions == (".yaml", ".yml") + + +def test_registry_digest_is_deterministic() -> None: + assert LanguageRegistry.bundled().digest() == LanguageRegistry.bundled().digest() diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py new file mode 100644 index 000000000..84c8d2c8e --- /dev/null +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -0,0 +1,140 @@ +"""Process-level merge, wheel, and real-consumer lifecycle scenarios.""" + +import os +import json +import subprocess +import sys +from pathlib import Path, PurePosixPath + +import pytest + +from pdd.sync_core import ( + PlannedWrite, + TransactionConflict, + TransactionManager, + build_unit_manifest, +) +from pdd.sync_core.identity import initialize_repository_identity + + +def _run(root: Path, *args: str, env=None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=root, + capture_output=True, + text=True, + check=False, + env=env, + ) + + +def _git(root: Path, *args: str) -> str: + result = _run(root, "git", *args) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +def _record_metric(name: str, value: int) -> None: + configured = os.environ.get("PDD_LIFECYCLE_METRICS_PATH") + if not configured: + return + path = Path(configured) + payload = json.loads(path.read_text()) if path.exists() else {} + payload[name] = value + path.write_text(json.dumps(payload, sort_keys=True)) + + +def test_merge_base_movement_blocks_stale_repair_and_converges(tmp_path) -> None: + """A repair planned before merge movement must never overwrite merged bytes.""" + root = tmp_path / "merge-repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "sync@example.com") + _git(root, "config", "user.name", "Sync Test") + initialize_repository_identity(root, "a1970bc0-ecde-4c13-b3dc-ee2fccf1d043") + (root / "prompts").mkdir() + (root / "src").mkdir() + (root / "prompts/widget_python.prompt").write_text("REQ-1: widget\n") + target = root / "src/widget.py" + target.write_text("value = 1\n") + (root / "architecture.json").write_text( + '[{"filename":"widget_python.prompt","filepath":"src/widget.py"}]\n' + ) + _git(root, "add", ".") + _git(root, "commit", "-qm", "protected base") + base = _git(root, "rev-parse", "HEAD") + + manager = TransactionManager(root) + repair = ( + PlannedWrite(PurePosixPath("src/widget.py"), b"value = 2\n", "100644"), + ) + manager.prepare("stale-repair", repair) + target.write_text("merged = True\n") + _git(root, "add", "src/widget.py") + _git(root, "commit", "-qm", "merge group moved") + head = _git(root, "rev-parse", "HEAD") + assert head != base + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + assert manifest.base_ref == base + assert manifest.head_ref == head + with pytest.raises(TransactionConflict, match="destination changed"): + manager.commit("stale-repair") + assert target.read_text() == "merged = True\n" + + fresh = TransactionManager(root) + fresh.prepare("fresh-repair", repair) + fresh.commit("fresh-repair") + before_second = _git(root, "status", "--porcelain", "--untracked-files=all") + second = fresh.prepare("second-run", repair) + assert second.no_op is True + after_second = _git(root, "status", "--porcelain", "--untracked-files=all") + _record_metric( + "post_merge_tree_changes", + int(before_second != after_second) + len(second.changed_paths), + ) + + +@pytest.fixture(scope="module") +def released_checker(tmp_path_factory) -> Path: + """Build and install the exact wheel used by clean-environment scenarios.""" + tmp_path = tmp_path_factory.mktemp("released-checker") + root = Path(__file__).resolve().parents[1] + dist = tmp_path / "dist" + build = _run(root, sys.executable, "-m", "build", "--wheel", "--outdir", str(dist)) + assert build.returncode == 0, build.stdout + build.stderr + wheels = tuple(dist.glob("*.whl")) + assert len(wheels) == 1 + wheel = wheels[0] + environment = tmp_path / "venv" + create = _run( + root, + sys.executable, + "-m", + "venv", + "--system-site-packages", + str(environment), + ) + assert create.returncode == 0, create.stderr + python = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + install = _run( + root, str(python), "-m", "pip", "install", "--no-deps", str(wheel) + ) + assert install.returncode == 0, install.stdout + install.stderr + return python + + +def test_built_wheel_runs_in_clean_virtual_environment( + tmp_path, released_checker +) -> None: + """The packaged checker and protected registry work outside the source tree.""" + probe = _run( + tmp_path, + str(released_checker), + "-c", + ( + "from pdd.sync_core import LanguageRegistry; " + "r=LanguageRegistry.bundled(); " + "assert r.resolve_alias('python').language_id == 'python'" + ), + ) + assert probe.returncode == 0, probe.stdout + probe.stderr diff --git a/tests/test_sync_core_manifest.py b/tests/test_sync_core_manifest.py new file mode 100644 index 000000000..80bf17ad5 --- /dev/null +++ b/tests/test_sync_core_manifest.py @@ -0,0 +1,437 @@ +"""Lifecycle tests for complete base/head candidate inventory.""" + +import json +import subprocess +from pathlib import Path, PurePosixPath + +from pdd.sync_core import InventoryStatus, build_unit_manifest +from pdd.sync_core.identity import initialize_repository_identity +from pdd.sync_core.language import LanguageRegistry, LanguageSpec + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def _commit(root: Path, message: str) -> str: + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", message) + return _git(root, "rev-parse", "HEAD") + + +def _repository(tmp_path: Path) -> Path: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "sync@example.com") + _git(root, "config", "user.name", "Sync Test") + initialize_repository_identity( + root, "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + ) + (root / "prompts").mkdir() + (root / "src").mkdir() + (root / "prompts/widget_python.prompt").write_text("Build widget\n") + (root / "src/widget.py").write_text("value = 1\n") + (root / "README.md").write_text("Human documentation\n") + (root / ".pdd/sync-ownership.json").write_text( + json.dumps( + { + "rules": [ + { + "pattern": "README.md", + "inventory": "HUMAN_OWNED", + "role": "documentation", + "owner": "docs@example.com", + } + ] + } + ) + ) + (root / "architecture.json").write_text( + json.dumps( + [{"filename": "widget_python.prompt", "filepath": "src/widget.py"}] + ) + ) + return root + + +def test_manifest_partitions_every_base_and_head_path(tmp_path) -> None: + root = _repository(tmp_path) + base = _commit(root, "base") + (root / "prompts/helper_python.prompt").write_text("Build helper\n") + head = _commit(root, "head") + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + tracked = set(_git(root, "ls-tree", "-r", "--name-only", head).splitlines()) + assert {item.candidate_id.artifact_relpath.as_posix() for item in manifest.candidates} == tracked + assert manifest.unaccounted_tracked_paths == () + assert len(manifest.expected_managed) == 2 + assert len(manifest.managed_units) == 2 + readme = next( + item for item in manifest.candidates if item.candidate_id.artifact_relpath.as_posix() == "README.md" + ) + assert readme.inventory is InventoryStatus.HUMAN_OWNED + + +def test_removed_prompt_remains_expected_and_requires_tombstone(tmp_path) -> None: + root = _repository(tmp_path) + base = _commit(root, "base") + (root / "prompts/widget_python.prompt").unlink() + head = _commit(root, "remove prompt") + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + assert len(manifest.expected_managed) == 1 + assert manifest.managed_units == () + assert any("lacks a complete" in reason for reason in manifest.invalid_reasons) + + +def test_protected_tombstone_accounts_for_removed_unit(tmp_path) -> None: + root = _repository(tmp_path) + base = _commit(root, "base") + (root / "prompts/widget_python.prompt").unlink() + tombstone = root / ".pdd/sync-tombstones.json" + tombstone.write_text( + json.dumps( + [ + { + "prompt_path": "prompts/widget_python.prompt", + "artifact_paths": [ + "prompts/widget_python.prompt", + "src/widget.py", + ], + "rationale": "Widget was deliberately retired", + "owner": "sync-owner@example.com", + "baseline_status": "IN_SYNC", + } + ] + ) + ) + head = _commit(root, "decommission widget") + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + assert not any("lacks a complete" in reason for reason in manifest.invalid_reasons) + assert manifest.units[0].tombstoned is True + assert len(manifest.expected_managed) == 1 + + +def test_incomplete_tombstone_cannot_reduce_expected_managed(tmp_path) -> None: + root = _repository(tmp_path) + base = _commit(root, "base") + (root / "prompts/widget_python.prompt").unlink() + (root / ".pdd/sync-tombstones.json").write_text( + json.dumps( + [ + { + "prompt_path": "prompts/widget_python.prompt", + "artifact_paths": ["prompts/widget_python.prompt"], + "rationale": "Attempted debt removal", + "owner": "candidate", + "baseline_status": "DRIFTED", + } + ] + ) + ) + head = _commit(root, "incomplete tombstone") + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + assert len(manifest.expected_managed) == 1 + assert any("complete IN_SYNC tombstone" in item for item in manifest.invalid_reasons) + + +def test_architecture_mapping_prefers_project_prompt_root_over_duplicate_leaf( + tmp_path, +) -> None: + root = _repository(tmp_path) + nested = root / "prompts/nested" + nested.mkdir() + (nested / "widget_python.prompt").write_text("Build another widget\n") + commit = _commit(root, "ambiguous prompt") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + assert not any("has 2 matches" in reason for reason in manifest.invalid_reasons) + widget = next( + unit + for unit in manifest.managed_units + if unit.unit_id.prompt_relpath.as_posix() == "prompts/widget_python.prompt" + ) + assert widget.artifact_paths == (Path("src/widget.py"),) + + +def test_unmatched_tracked_path_is_unaccounted_not_default_human(tmp_path) -> None: + root = _repository(tmp_path) + (root / "src/orphan.py").write_text("orphan = True\n") + commit = _commit(root, "orphan") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + assert [path.as_posix() for path in manifest.unaccounted_tracked_paths] == [ + "src/orphan.py" + ] + orphan = next( + item + for item in manifest.candidates + if item.candidate_id.artifact_relpath.as_posix() == "src/orphan.py" + ) + assert orphan.inventory is InventoryStatus.INVALID + + +def test_protected_human_rule_precedes_prompt_filename_inference(tmp_path) -> None: + root = _repository(tmp_path) + policy = json.loads((root / ".pdd/sync-ownership.json").read_text()) + policy["rules"].append( + { + "pattern": "tests/fixtures/**", + "inventory": "HUMAN_OWNED", + "role": "test-fixture", + "owner": "quality@example.com", + } + ) + (root / ".pdd/sync-ownership.json").write_text(json.dumps(policy)) + fixture = root / "tests/fixtures/sample_python.prompt" + fixture.parent.mkdir(parents=True) + fixture.write_text("This resembles a prompt but is test data\n") + commit = _commit(root, "human prompt fixture") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + assert len(manifest.managed_units) == 1 + candidate = next( + item + for item in manifest.candidates + if item.candidate_id.artifact_relpath.as_posix() + == "tests/fixtures/sample_python.prompt" + ) + assert candidate.inventory is InventoryStatus.HUMAN_OWNED + + +def test_manifest_digest_ignores_refs_and_dynamic_canonical_state(tmp_path) -> None: + root = _repository(tmp_path) + base = _commit(root, "base") + first = build_unit_manifest(root, base_ref=base, head_ref=base) + state = root / ".pdd/meta/v2/fingerprint.json" + state.parent.mkdir(parents=True) + state.write_text("{}\n") + head = _commit(root, "canonical state") + second = build_unit_manifest(root, base_ref=base, head_ref=head) + assert first.digest() == second.digest() + + +def test_worktree_repository_id_cannot_change_immutable_manifest_identity( + tmp_path, +) -> None: + root = _repository(tmp_path) + commit = _commit(root, "base") + (root / ".pdd/repository-id").write_text( + "11111111-1111-4111-8111-111111111111\n" + ) + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + assert manifest.repository_id == "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + + +def test_duplicate_architecture_output_is_invalid(tmp_path) -> None: + root = _repository(tmp_path) + architecture = json.loads((root / "architecture.json").read_text()) + architecture.append(dict(architecture[0])) + (root / "architecture.json").write_text(json.dumps(architecture)) + commit = _commit(root, "duplicate output") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + assert any("duplicate output" in reason for reason in manifest.invalid_reasons) + + +def test_architecture_resolves_prompt_from_repository_prompt_root(tmp_path) -> None: + root = _repository(tmp_path) + nested_prompt = root / "prompts/backend/widget_python.prompt" + nested_prompt.parent.mkdir() + nested_prompt.write_text("Build backend widget\n") + architecture = json.loads((root / "architecture.json").read_text()) + architecture.append( + { + "filename": "backend/widget_python.prompt", + "filepath": "src/backend_widget.py", + } + ) + (root / "architecture.json").write_text(json.dumps(architecture)) + commit = _commit(root, "repository prompt root") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + backend = next( + unit + for unit in manifest.managed_units + if unit.unit_id.prompt_relpath == PurePosixPath( + "prompts/backend/widget_python.prompt" + ) + ) + assert backend.artifact_paths == (PurePosixPath("src/backend_widget.py"),) + + +def test_excluded_project_architecture_is_not_a_canonical_declaration(tmp_path) -> None: + root = _repository(tmp_path) + policy = json.loads((root / ".pdd/sync-ownership.json").read_text()) + policy["rules"].append( + { + "pattern": "vendor/**", + "inventory": "HUMAN_OWNED", + "role": "excluded-project", + "owner": "vendor@example.com", + } + ) + (root / ".pdd/sync-ownership.json").write_text(json.dumps(policy)) + vendor = root / "vendor" + vendor.mkdir() + (vendor / "architecture.json").write_text( + json.dumps( + [{"filename": "missing_python.prompt", "filepath": "missing.py"}] + ) + ) + commit = _commit(root, "excluded nested project") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + assert not any("vendor/architecture.json" in item for item in manifest.invalid_reasons) + architecture = next( + item + for item in manifest.candidates + if item.candidate_id.artifact_relpath + == PurePosixPath("vendor/architecture.json") + ) + assert architecture.inventory is InventoryStatus.HUMAN_OWNED + + +def test_nested_architecture_output_is_relative_to_project_root(tmp_path) -> None: + root = _repository(tmp_path) + project = root / "project" + (project / "prompts").mkdir(parents=True) + (project / "src").mkdir() + (project / "prompts/nested_python.prompt").write_text("Build nested\n") + (project / "src/nested.py").write_text("nested = True\n") + (project / "architecture.json").write_text( + json.dumps( + [ + { + "filename": "nested_python.prompt", + "filepath": "src/nested.py", + } + ] + ) + ) + commit = _commit(root, "nested architecture") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + nested = next( + unit + for unit in manifest.managed_units + if unit.unit_id.prompt_relpath + == PurePosixPath("project/prompts/nested_python.prompt") + ) + assert nested.artifact_paths == (PurePosixPath("project/src/nested.py"),) + + +def test_architecture_owned_prompt_output_is_not_a_second_source_unit(tmp_path) -> None: + root = _repository(tmp_path) + project = root / "project" + (project / "prompts").mkdir(parents=True) + (project / "src").mkdir() + source = project / "prompts/generated_python.prompt" + output = project / "src/generated_python.prompt" + source.write_text("Generate the runtime prompt\n") + output.write_text("Runtime instructions\n") + (project / "architecture.json").write_text( + json.dumps( + [ + { + "filename": "generated_python.prompt", + "filepath": "src/generated_python.prompt", + } + ] + ) + ) + commit = _commit(root, "generated prompt artifact") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + prompt_paths = {unit.unit_id.prompt_relpath for unit in manifest.managed_units} + assert PurePosixPath("project/prompts/generated_python.prompt") in prompt_paths + assert PurePosixPath("project/src/generated_python.prompt") not in prompt_paths + generated = next( + unit + for unit in manifest.managed_units + if unit.unit_id.prompt_relpath + == PurePosixPath("project/prompts/generated_python.prompt") + ) + assert generated.artifact_paths == ( + PurePosixPath("project/src/generated_python.prompt"), + ) + + +def test_unmapped_prompt_uses_protected_config_output_template(tmp_path) -> None: + root = _repository(tmp_path) + (root / ".pddrc").write_text( + "version: '1.0'\n" + "contexts:\n" + " python:\n" + " defaults:\n" + " prompts_dir: prompts\n" + " generate_output_path: src\n" + " outputs:\n" + " code:\n" + " path: src/{name}.py\n" + ) + (root / "prompts/helper_python.prompt").write_text("Build helper\n") + (root / "src/helper.py").write_text("helper = True\n") + commit = _commit(root, "configured helper") + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + helper = next( + unit + for unit in manifest.managed_units + if unit.unit_id.prompt_relpath == PurePosixPath("prompts/helper_python.prompt") + ) + assert helper.artifact_paths == (PurePosixPath("src/helper.py"),) + + +def test_manifest_digest_binds_protected_config_blob_bytes(tmp_path) -> None: + root = _repository(tmp_path) + config = root / ".pddrc" + config.write_text( + "version: '1.0'\ncontexts:\n default:\n defaults:\n" + " prompts_dir: prompts\n generate_output_path: src\n" + ) + first_commit = _commit(root, "first config") + first = build_unit_manifest(root, base_ref=first_commit, head_ref=first_commit) + config.write_text(config.read_text() + " strength: 0.5\n") + second_commit = _commit(root, "change config bytes") + second = build_unit_manifest(root, base_ref=second_commit, head_ref=second_commit) + assert first.digest() != second.digest() + + +def test_manifest_digest_binds_language_registry_policy(tmp_path) -> None: + root = _repository(tmp_path) + commit = _commit(root, "base") + first_registry = LanguageRegistry( + (LanguageSpec("python", "Python", ("python",), (".py",), ("code",)),) + ) + second_registry = LanguageRegistry( + ( + LanguageSpec( + "python", "Python", ("python",), (".py",), ("code", "test") + ), + ) + ) + first = build_unit_manifest( + root, base_ref=commit, head_ref=commit, registry=first_registry + ) + second = build_unit_manifest( + root, base_ref=commit, head_ref=commit, registry=second_registry + ) + assert first.digest() != second.digest() + + +def test_protected_human_pattern_does_not_auto_classify_new_head_path(tmp_path) -> None: + root = _repository(tmp_path) + policy = json.loads((root / ".pdd/sync-ownership.json").read_text()) + policy["rules"].append( + { + "pattern": "docs/**", + "inventory": "HUMAN_OWNED", + "role": "documentation", + "owner": "docs@example.com", + } + ) + (root / ".pdd/sync-ownership.json").write_text(json.dumps(policy)) + (root / "docs").mkdir() + (root / "docs/existing.md").write_text("protected human file\n") + base = _commit(root, "human base") + (root / "docs/new.md").write_text("candidate addition\n") + head = _commit(root, "new file") + manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + assert PurePosixPath("docs/new.md") in manifest.unaccounted_tracked_paths diff --git a/tests/test_sync_core_planner_capabilities.py b/tests/test_sync_core_planner_capabilities.py new file mode 100644 index 000000000..33fd1ad7d --- /dev/null +++ b/tests/test_sync_core_planner_capabilities.py @@ -0,0 +1,107 @@ +"""Tests for fail-closed command capabilities and repair planning.""" + +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + BaselineStatus, + CandidateId, + CapabilityError, + CapabilityRegistry, + InventoryStatus, + RepairAction, + RepairPolicy, + SemanticStatus, + SyncVerdict, + plan_repair, +) +from pdd.sync_core.types import VerdictDetails + + +SUBJECT = CandidateId("repository-1", PurePosixPath("prompts/widget.prompt"), "prompt") + + +def _verdict(baseline, semantic, changed=(), missing=()): + return SyncVerdict( + SUBJECT, + InventoryStatus.MANAGED, + baseline, + semantic, + VerdictDetails(changed, "test verdict", missing), + ) + + +def test_reconcile_and_certify_are_strictly_read_only() -> None: + registry = CapabilityRegistry.protected_defaults() + assert registry.get("reconcile").read_only is True + assert registry.get("certify").read_only is True + with pytest.raises(CapabilityError, match="cannot write roles"): + registry.authorize_writes("reconcile", {"fingerprint"}) + + +def test_unknown_command_and_out_of_scope_write_fail_closed() -> None: + registry = CapabilityRegistry.protected_defaults() + with pytest.raises(CapabilityError, match="unknown"): + registry.get("candidate-command") + with pytest.raises(CapabilityError, match="cannot write roles"): + registry.authorize_writes("baseline", {"code"}) + + +def test_mutating_commands_require_transactions() -> None: + registry = CapabilityRegistry.protected_defaults() + for command in ("baseline", "sync", "update", "resolve", "trusted-validator"): + assert registry.get(command).requires_transaction is True + assert registry.get("baseline").may_reset_baseline is True + assert registry.get("trusted-validator").may_issue_evidence is True + + +@pytest.mark.parametrize("role", ["prompt", "include", "manifest"]) +def test_prompt_side_drift_can_only_plan_generation(role) -> None: + verdict = _verdict(BaselineStatus.DRIFTED, SemanticStatus.UNKNOWN, (role,)) + plan = plan_repair(verdict, RepairPolicy()) + assert plan.action is RepairAction.SYNC_FROM_PROMPT + assert plan.unattended is True + assert plan.verdict is verdict + + +@pytest.mark.parametrize("role", ["code", "test", "example"]) +def test_derived_drift_requires_reviewed_requirement_preserving_update(role) -> None: + plan = plan_repair( + _verdict(BaselineStatus.DRIFTED, SemanticStatus.UNKNOWN, (role,)), + RepairPolicy(), + ) + assert plan.action is RepairAction.UPDATE_PROMPT_REVIEW + assert plan.unattended is False + assert plan.review_required is True + + +def test_conflict_never_has_an_unattended_action() -> None: + plan = plan_repair( + _verdict(BaselineStatus.DRIFTED, SemanticStatus.CONFLICT, ("prompt", "code")), + RepairPolicy(), + ) + assert plan.action is RepairAction.RESOLVE_CONFLICT + assert plan.unattended is False + + +def test_unbaselined_defaults_to_reviewed_unknown_baseline() -> None: + plan = plan_repair( + _verdict(BaselineStatus.UNBASELINED, SemanticStatus.UNKNOWN), RepairPolicy() + ) + assert plan.action is RepairAction.BASELINE_RESET + assert plan.review_required is True + + +def test_missing_artifact_is_restored_not_stamped() -> None: + plan = plan_repair( + _verdict( + BaselineStatus.DRIFTED, + SemanticStatus.FAILED, + ("code",), + ("code",), + ), + RepairPolicy(), + ) + assert plan.action is RepairAction.RESTORE_REQUIRED + assert plan.unattended is False diff --git a/tests/test_sync_core_released_checker.py b/tests/test_sync_core_released_checker.py new file mode 100644 index 000000000..86b7baf6a --- /dev/null +++ b/tests/test_sync_core_released_checker.py @@ -0,0 +1,94 @@ +"""Tests for pinned released-checker runtime provenance.""" + +import hashlib +from pathlib import Path +import zipfile + +import pytest + +from pdd.sync_core import CheckerIdentity, checker_identity_from_environment +from pdd.sync_core.released_checker import ( + ReleasedCheckerError, + validate_released_checker_runtime, +) + + +def _wheel(tmp_path: Path) -> tuple[Path, CheckerIdentity, Path, Path]: + wheel = tmp_path / "pdd-1.0-py3-none-any.whl" + member = "pdd/sync_core/released_checker.py" + content = b"exact installed checker bytes\n" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr(member, content) + prefix = tmp_path / "venv" + package = prefix / "lib/python3.12/site-packages" / member + package.parent.mkdir(parents=True) + package.write_bytes(content) + identity = CheckerIdentity( + hashlib.sha256(wheel.read_bytes()).hexdigest(), + "refs/tags/v1.0.0", + "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v1.0.0", + ) + return wheel, identity, prefix, package + + +def test_pinned_wheel_in_isolated_site_packages_is_accepted(tmp_path) -> None: + wheel, identity, prefix, package = _wheel(tmp_path) + validate_released_checker_runtime( + identity, + wheel, + package_path=package, + prefix=prefix, + base_prefix=tmp_path / "system", + ) + + +def test_modified_wheel_is_rejected(tmp_path) -> None: + wheel, identity, prefix, package = _wheel(tmp_path) + wheel.write_bytes(b"modified after pinning") + with pytest.raises(ReleasedCheckerError, match="digest does not match"): + validate_released_checker_runtime( + identity, + wheel, + package_path=package, + prefix=prefix, + base_prefix=tmp_path / "system", + ) + + +def test_source_checkout_import_is_rejected(tmp_path) -> None: + wheel, identity, prefix, _package = _wheel(tmp_path) + with pytest.raises(ReleasedCheckerError, match="source checkout"): + validate_released_checker_runtime( + identity, + wheel, + package_path=tmp_path / "venv/src/pdd/sync_core/released_checker.py", + prefix=prefix, + base_prefix=tmp_path / "system", + ) + + +def test_modified_installed_checker_is_rejected(tmp_path) -> None: + wheel, identity, prefix, package = _wheel(tmp_path) + package.write_bytes(b"modified installed checker\n") + with pytest.raises(ReleasedCheckerError, match="differs from wheel"): + validate_released_checker_runtime( + identity, + wheel, + package_path=package, + prefix=prefix, + base_prefix=tmp_path / "system", + ) + + +def test_global_identity_requires_released_entrypoint_marker(monkeypatch) -> None: + monkeypatch.setenv("PDD_RELEASED_CHECKER_WHEEL_SHA256", "c" * 64) + monkeypatch.setenv("PDD_RELEASED_CHECKER_REF", "refs/tags/v1.0.0") + monkeypatch.setenv( + "PDD_RELEASED_CHECKER_WORKFLOW_IDENTITY", + "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v1.0.0", + ) + monkeypatch.delenv("PDD_RELEASED_CHECKER_EXECUTION", raising=False) + with pytest.raises(ValueError, match="pdd-sync-checker"): + checker_identity_from_environment() + monkeypatch.setenv("PDD_RELEASED_CHECKER_EXECUTION", "1") + assert checker_identity_from_environment().wheel_sha256 == "c" * 64 diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py new file mode 100644 index 000000000..2e4672853 --- /dev/null +++ b/tests/test_sync_core_reporting.py @@ -0,0 +1,548 @@ +"""Lifecycle test for trusted canonical reporting through the real WAL.""" + +import base64 +import json +import os +import subprocess +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from unittest.mock import patch + +from click.testing import CliRunner +import pytest + +from pdd.sync_core import ( + AttestationBinding, + AttestationRequest, + AttestationSigner, + CanonicalReportOptions, + EvidenceOutcome, + FingerprintProvenance, + FingerprintRecord, + FingerprintStore, + PlannedWrite, + SemanticStatus, + TransactionManager, + TRUSTED_RUNNER_VERSION, + build_canonical_report, + build_unit_manifest, + build_unit_snapshot, + encode_attestation, + encode_fingerprint, + evidence_relpath, + finalize_unit, + load_verification_profiles, + runner_identity_digest, +) +from pdd.sync_core.identity import initialize_repository_identity +from pdd.sync_core.types import ObligationEvidence +from pdd.ci_drift_heal import main as ci_drift_heal_main +from pdd.commands.sync_core import baseline as baseline_command +from pdd.continuous_sync import build_report as build_compatibility_report +from pdd.continuous_sync import ( + canonical_sync_enabled, + project_root, + repository_root, +) +from pdd.operation_log import save_fingerprint + + +REPOSITORY_ID = "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" +NOW = datetime(2026, 7, 10, 12, 0, tzinfo=timezone.utc) +SIGNER = AttestationSigner("trusted-ci", b"r" * 32) + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _repository(tmp_path: Path) -> tuple[Path, str]: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "report@example.com") + _git(root, "config", "user.name", "Report Test") + initialize_repository_identity(root, REPOSITORY_ID) + for directory in ("prompts", "src", "docs", "tests"): + (root / directory).mkdir() + (root / "prompts/widget_python.prompt").write_text( + "REQ-1: Build widget\ndocs/widget.md\n" + ) + (root / "docs/widget.md").write_text("Widget contract\n") + (root / "src/widget.py").write_text("value = 1\n") + (root / "tests/test_widget.py").write_text("def test_widget(): assert True\n") + (root / "architecture.json").write_text( + json.dumps( + [{"filename": "widget_python.prompt", "filepath": "src/widget.py"}] + ) + ) + (root / ".pdd/sync-ownership.json").write_text( + json.dumps( + { + "rules": [ + { + "pattern": "docs/widget.md", + "inventory": "HUMAN_OWNED", + "role": "documentation", + "owner": "docs@example.com", + }, + { + "pattern": "tests/test_widget.py", + "inventory": "HUMAN_OWNED", + "role": "test", + "owner": "quality@example.com", + }, + ] + } + ) + ) + (root / ".pdd/verification-profiles.json").write_text( + json.dumps( + { + "profiles": [ + { + "prompt_path": "prompts/widget_python.prompt", + "language_id": "python", + "required_requirement_ids": ["REQ-1"], + "obligations": [ + { + "obligation_id": "pytest", + "kind": "test", + "validator_id": "pytest", + "validator_config_digest": "pytest-v1", + "requirement_ids": ["REQ-1"], + "artifact_paths": ["tests/test_widget.py"], + } + ], + } + ] + } + ) + ) + (root / ".pdd/attestation-trust.json").write_text( + json.dumps( + { + "issuers": [ + { + "issuer_id": "trusted-ci", + "public_key": base64.b64encode( + SIGNER.public_key_bytes() + ).decode("ascii"), + } + ], + "revoked_issuers": [], + "revoked_attestations": [], + } + ) + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "managed baseline") + return root, _git(root, "rev-parse", "HEAD") + + +def _finalize_trusted_baseline(root: Path, commit: str) -> None: + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + profile = load_verification_profiles(root, manifest).profiles[0] + unit = manifest.managed_units[0] + snapshot = build_unit_snapshot(root, manifest, unit, profile) + binding = AttestationBinding( + unit.unit_id, + snapshot.digest(), + profile.profile_digest, + runner_identity_digest(profile, root=root, ref=commit), + TRUSTED_RUNNER_VERSION, + commit, + commit, + ) + envelope = SIGNER.issue( + AttestationRequest( + "attestation-widget-1", + binding, + (ObligationEvidence("pytest", EvidenceOutcome.PASS),), + "nonce-widget-1", + NOW, + ) + ) + provenance = FingerprintProvenance( + "generated", + "pdd sync widget", + "transaction-widget-1", + commit, + NOW.isoformat(), + "pdd-test", + ) + record = FingerprintRecord( + snapshot, + 2, + 2, + provenance, + SemanticStatus.VERIFIED, + envelope.attestation_id, + ) + store = FingerprintStore(root) + fingerprint_path = store.path_for(unit.unit_id).relative_to(root) + writes = ( + PlannedWrite( + evidence_relpath(envelope.attestation_id), + encode_attestation(envelope), + "100644", + ), + PlannedWrite( + PurePosixPath(fingerprint_path.as_posix()), + encode_fingerprint(record), + "100644", + ), + ) + manager = TransactionManager(root) + manager.prepare("transaction-widget-1", writes) + manager.commit("transaction-widget-1") + + +def _options(tmp_path: Path, commit: str) -> CanonicalReportOptions: + return CanonicalReportOptions( + base_ref=commit, + head_ref=commit, + replay_ledger_path=tmp_path / "external-trust/replay.json", + now=NOW, + ) + + +def test_trusted_transactional_baseline_passes_canonical_predicate(tmp_path) -> None: + root, commit = _repository(tmp_path) + _finalize_trusted_baseline(root, commit) + report = build_canonical_report(root, _options(tmp_path, commit)) + assert report["ok"] is True + assert report["counts"]["managed_units"] == 1 + assert report["counts"]["trusted_in_sync"] == 1 + assert report["counts"]["unaccounted_tracked_paths"] == 0 + assert report["units"][0]["in_sync"] is True + + +def test_managed_waiver_is_counted_and_blocks_certificate_predicate(tmp_path) -> None: + root, _commit = _repository(tmp_path) + (root / ".pdd/sync-waivers.json").write_text( + json.dumps( + { + "waivers": [ + { + "waiver_id": "SYNC-1", + "prompt_path": "prompts/widget_python.prompt", + "snapshot_digest": "a" * 64, + "approved_by": "reviewer@example.com", + "reason": "temporary reviewed exception", + "expires_at": "2026-08-01T00:00:00+00:00", + } + ] + } + ) + ) + _git(root, "add", ".pdd/sync-waivers.json") + _git(root, "commit", "-q", "-m", "protected waiver") + commit = _git(root, "rev-parse", "HEAD") + report = build_canonical_report(root, _options(tmp_path, commit)) + assert report["counts"]["managed_waivers"] == 1 + assert report["ok"] is False + + +def test_candidate_cannot_delete_protected_sync_waiver(tmp_path) -> None: + root, _commit = _repository(tmp_path) + waiver = root / ".pdd/sync-waivers.json" + waiver.write_text( + json.dumps( + { + "waivers": [ + { + "waiver_id": "SYNC-1", + "prompt_path": "prompts/widget_python.prompt", + "snapshot_digest": "b" * 64, + "approved_by": "reviewer@example.com", + "reason": "protected exception", + "expires_at": "2026-08-01T00:00:00+00:00", + } + ] + } + ) + ) + _git(root, "add", ".pdd/sync-waivers.json") + _git(root, "commit", "-q", "-m", "protected waiver") + base = _git(root, "rev-parse", "HEAD") + waiver.unlink() + _git(root, "add", ".pdd/sync-waivers.json") + _git(root, "commit", "-q", "-m", "remove waiver") + head = _git(root, "rev-parse", "HEAD") + options = CanonicalReportOptions( + base_ref=base, + head_ref=head, + replay_ledger_path=tmp_path / "external-trust/removal.json", + now=NOW, + ) + report = build_canonical_report(root, options) + assert report["counts"]["managed_waivers"] == 1 + assert any("removed protected waiver" in item for item in report["errors"]) + + +def test_nested_config_cannot_bypass_repository_canonical_finalizer( + tmp_path, monkeypatch +) -> None: + root, _commit = _repository(tmp_path) + nested = root / "nested" + prompts = nested / "prompts" + prompts.mkdir(parents=True) + (nested / ".pddrc").write_text("[pdd]\n") + prompt = prompts / "worker_python.prompt" + prompt.write_text("REQ-1: Build nested worker\n") + _git(root, "add", "nested") + _git(root, "commit", "-q", "-m", "nested project") + head = _git(root, "rev-parse", "HEAD") + monkeypatch.setenv("PDD_SYNC_PROTECTED_BASE_SHA", head) + + assert project_root(prompt) == nested + assert repository_root(prompt) == root + assert canonical_sync_enabled(nested) is True + signer = object() + with patch( + "pdd.sync_core.finalize.attestation_signer_from_environment", + return_value=signer, + ), patch("pdd.sync_core.finalize.finalize_unit") as mocked_finalize: + save_fingerprint( + "worker", + "python", + "generate", + {"prompt": prompt}, + ) + mocked_finalize.assert_called_once_with( + root, + PurePosixPath("nested/prompts/worker_python.prompt"), + base_ref=head, + head_ref="HEAD", + signer=signer, + ) + + +def test_code_drift_cannot_reuse_old_attestation(tmp_path) -> None: + root, commit = _repository(tmp_path) + _finalize_trusted_baseline(root, commit) + (root / "src/widget.py").write_text("value = 2\n") + report = build_canonical_report(root, _options(tmp_path, commit)) + assert report["ok"] is False + assert report["counts"]["drifted"] == 1 + assert report["counts"]["trusted_in_sync"] == 0 + + +def test_forged_candidate_evidence_is_rejected(tmp_path) -> None: + root, commit = _repository(tmp_path) + _finalize_trusted_baseline(root, commit) + evidence_path = root.joinpath(*evidence_relpath("attestation-widget-1").parts) + payload = json.loads(evidence_path.read_text()) + payload["binding"]["checked_sha"] = "forged-head" + evidence_path.write_text(json.dumps(payload)) + report = build_canonical_report(root, _options(tmp_path, commit)) + assert report["ok"] is False + assert report["counts"]["failed"] == 1 + assert "signature is invalid" in report["units"][0]["reason"] + + +def test_live_ci_uses_canonical_gate_and_fails_on_drift(tmp_path, monkeypatch) -> None: + root, commit = _repository(tmp_path) + _finalize_trusted_baseline(root, commit) + monkeypatch.chdir(root) + monkeypatch.setenv( + "PDD_ATTESTATION_REPLAY_LEDGER", + str(tmp_path / "external-trust/replay.json"), + ) + monkeypatch.setattr("pdd.sync_core.reporting.datetime", type("Clock", (), {"now": staticmethod(lambda _tz: NOW)})) + assert ci_drift_heal_main(skip_ci=True) == 0 + (root / "src/widget.py").write_text("value = 3\n") + assert ci_drift_heal_main(skip_ci=True) == 1 + + +def test_legacy_report_consumer_cannot_self_certify_matching_v1_hashes( + tmp_path, monkeypatch +) -> None: + root = tmp_path / "canonical" + (root / ".pdd").mkdir(parents=True) + (root / ".pdd/repository-id").write_text(REPOSITORY_ID + "\n") + canonical = { + "ok": False, + "project_root": str(root), + "counts": { + "managed_units": 1, + "trusted_in_sync": 0, + "drifted": 0, + "unbaselined": 0, + "corrupt": 0, + "unknown": 1, + "conflict": 0, + "failed": 0, + "invalid": 0, + }, + "units": [ + { + "subject": "prompts/widget_python.prompt", + "baseline": "CURRENT", + "semantic": "UNKNOWN", + "in_sync": False, + "reason": "legacy fingerprint has no trusted evidence", + "changed_roles": [], + } + ], + } + monkeypatch.setattr( + "pdd.continuous_sync.build_canonical_report", lambda *_a, **_k: canonical + ) + monkeypatch.setattr("pdd.continuous_sync.canonical_sync_enabled", lambda _root: True) + report = build_compatibility_report(consumer="sync-dry-run", root=root) + assert report["ok"] is False + assert report["summary"]["synced"] == 0 + assert report["summary"]["failures"] == 1 + assert report["units"][0]["classification"] == "FAILURE" + + +def test_reviewed_baseline_command_records_current_unknown(tmp_path, monkeypatch) -> None: + root, commit = _repository(tmp_path) + monkeypatch.chdir(root) + result = CliRunner().invoke( + baseline_command, + [ + "--module", + "prompts/widget_python.prompt", + "--reviewed-by", + "reviewer@example.com", + "--reason", + "adopt existing bytes before trusted validation", + ], + ) + assert result.exit_code == 0, result.output + report = build_canonical_report(root, _options(tmp_path, commit)) + assert report["ok"] is False + assert report["units"][0]["baseline"] == "CURRENT" + assert report["units"][0]["semantic"] == "UNKNOWN" + assert report["counts"]["trusted_in_sync"] == 0 + + +def test_trusted_finalizer_commits_artifact_closure_evidence_and_fingerprint( + tmp_path, +) -> None: + root, commit = _repository(tmp_path) + result = finalize_unit( + root, + PurePosixPath("prompts/widget_python.prompt"), + base_ref=commit, + head_ref=commit, + signer=SIGNER, + replay_ledger_path=tmp_path / "external-trust/finalizer.json", + ) + assert result.transaction.phase.value == "COMMITTED" + _git(root, "add", ".pdd/meta/v2", ".pdd/evidence/v2") + _git(root, "commit", "-q", "-m", "commit trusted sync evidence") + finalized_commit = _git(root, "rev-parse", "HEAD") + options = CanonicalReportOptions( + base_ref=commit, + head_ref=finalized_commit, + replay_ledger_path=tmp_path / "external-trust/finalize-replay.json", + ) + report = build_canonical_report(root, options) + assert report["ok"] is True + assert report["counts"]["trusted_in_sync"] == 1 + + +def test_trusted_finalizer_second_run_is_zero_write_no_op(tmp_path) -> None: + root, commit = _repository(tmp_path) + replay = tmp_path / "external-trust/idempotency.json" + first = finalize_unit( + root, + PurePosixPath("prompts/widget_python.prompt"), + base_ref=commit, + head_ref=commit, + signer=SIGNER, + replay_ledger_path=replay, + ) + state_roots = (root / ".pdd/meta/v2", root / ".pdd/evidence/v2") + before = { + path.relative_to(root): path.read_bytes() + for state_root in state_roots + for path in state_root.rglob("*") + if path.is_file() + } + with patch("pdd.sync_core.finalize.run_profile") as run_profile_mock: + second = finalize_unit( + root, + PurePosixPath("prompts/widget_python.prompt"), + base_ref=commit, + head_ref=commit, + signer=SIGNER, + replay_ledger_path=replay, + ) + after = { + path.relative_to(root): path.read_bytes() + for state_root in state_roots + for path in state_root.rglob("*") + if path.is_file() + } + assert second.transaction.no_op is True + assert second.attestation_id == first.attestation_id + assert before == after + run_profile_mock.assert_not_called() + metrics_path = os.environ.get("PDD_LIFECYCLE_METRICS_PATH") + if metrics_path: + path = Path(metrics_path) + payload = json.loads(path.read_text()) if path.exists() else {} + payload["post_repair_second_run_writes"] = ( + len(second.transaction.changed_paths) + int(before != after) + ) + path.write_text(json.dumps(payload, sort_keys=True)) + + +@pytest.mark.parametrize( + ("edits", "semantic", "changed_roles"), + [ + (("prompt",), "UNKNOWN", ["prompt"]), + (("code",), "UNKNOWN", ["code"]), + (("test",), "UNKNOWN", ["test"]), + (("include",), "UNKNOWN", ["include"]), + (("prompt", "code"), "CONFLICT", ["code", "prompt"]), + ], +) +def test_canonical_source_edit_matrix_detects_each_channel( + tmp_path, edits, semantic, changed_roles +) -> None: + root, commit = _repository(tmp_path) + _finalize_trusted_baseline(root, commit) + paths = { + "prompt": root / "prompts/widget_python.prompt", + "code": root / "src/widget.py", + "test": root / "tests/test_widget.py", + "include": root / "docs/widget.md", + } + for role in edits: + with paths[role].open("a", encoding="utf-8") as handle: + handle.write(f"# changed {role}\n") + report = build_canonical_report(root, _options(tmp_path, commit)) + assert report["ok"] is False + assert report["units"][0]["baseline"] == "DRIFTED" + assert report["units"][0]["semantic"] == semantic + assert report["units"][0]["changed_roles"] == changed_roles + + +def test_trusted_finalizer_rejects_test_time_artifact_mutation(tmp_path) -> None: + root, _commit = _repository(tmp_path) + (root / "tests/test_widget.py").write_text( + "from pathlib import Path\n" + "def test_widget():\n" + " Path('src/widget.py').write_text('mutated = True\\n')\n" + ) + _git(root, "add", "tests/test_widget.py") + _git(root, "commit", "-q", "-m", "protected mutation probe") + head = _git(root, "rev-parse", "HEAD") + with pytest.raises(ValueError, match="changed during trusted validation"): + finalize_unit( + root, + PurePosixPath("prompts/widget_python.prompt"), + base_ref=head, + head_ref=head, + signer=SIGNER, + replay_ledger_path=tmp_path / "external-trust/mutation.json", + ) + assert not (root / ".pdd/evidence/v2").exists() diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py new file mode 100644 index 000000000..fda8d6ead --- /dev/null +++ b/tests/test_sync_core_runner.py @@ -0,0 +1,229 @@ +"""Tests for pass-only trusted runner normalization and self-certification guards.""" + +import subprocess +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + +from pdd.sync_core import ( + AttestationSigner, + AttestationIssue, + EvidenceOutcome, + RunnerConfig, + RunBinding, + UnitId, + VerificationObligation, + VerificationProfile, + run_profile, +) + + +UNIT = UnitId("repository-1", PurePosixPath("prompts/widget_python.prompt"), "python") + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _repository(tmp_path: Path, test_content: str) -> tuple[Path, str]: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "runner@example.com") + _git(root, "config", "user.name", "Runner Test") + (root / "tests").mkdir() + (root / "tests/test_widget.py").write_text(test_content) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "tests") + return root, _git(root, "rev-parse", "HEAD") + + +def _profile() -> VerificationProfile: + obligation = VerificationObligation( + "pytest", + "test", + "pytest", + "pytest-v1", + ("REQ-1",), + (PurePosixPath("tests/test_widget.py"),), + ) + return VerificationProfile(UNIT, (obligation,), ("REQ-1",), "profile-v1") + + +def _run(root: Path, base: str, head: str): + return run_profile( + root, + _profile(), + RunBinding("snapshot-v1", base, head), + AttestationIssue( + AttestationSigner("trusted-ci", b"v" * 32), + "attestation-1", + "nonce-1", + datetime(2026, 7, 10, 12, 0, tzinfo=timezone.utc), + ), + config=RunnerConfig(timeout_seconds=20), + ) + + +def test_passing_collected_test_is_pass(tmp_path) -> None: + root, commit = _repository(tmp_path, "def test_widget(): assert True\n") + envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is EvidenceOutcome.PASS + assert envelope.results[0].outcome is EvidenceOutcome.PASS + + +def test_zero_tests_is_not_collected(tmp_path) -> None: + root, commit = _repository(tmp_path, "value = 1\n") + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is EvidenceOutcome.NOT_COLLECTED + + +def test_skipped_test_cannot_satisfy_obligation(tmp_path) -> None: + root, commit = _repository( + tmp_path, + "import pytest\n@pytest.mark.skip(reason='not ready')\ndef test_widget(): pass\n", + ) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is EvidenceOutcome.SKIP + + +def test_candidate_modified_test_cannot_self_certify(tmp_path) -> None: + root, base = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/test_widget.py").write_text("def test_widget(): assert 1 == 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "weaken test") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + + +def test_worktree_modified_test_cannot_self_certify(tmp_path) -> None: + root, head = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/test_widget.py").write_text("def test_widget(): assert 1 == 1\n") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + + +def test_candidate_modified_conftest_cannot_self_certify(tmp_path) -> None: + root, base = _repository(tmp_path, "def test_widget(value): assert value == 1\n") + (root / "tests/conftest.py").write_text( + "import pytest\n@pytest.fixture\ndef value(): return 1\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate fixture") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "tests/conftest.py" in executions[0].detail + + +def test_untracked_conftest_cannot_influence_checked_sha(tmp_path) -> None: + root, head = _repository(tmp_path, "def test_widget(): assert False\n") + (root / "conftest.py").write_text( + "def pytest_collection_modifyitems(items):\n" + " for item in items: item.obj = lambda: None\n" + ) + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "conftest.py" in executions[0].detail + + +def test_candidate_modified_imported_test_helper_cannot_self_certify(tmp_path) -> None: + root, base = _repository( + tmp_path, + "from tests.helper import expected\ndef test_widget(): assert expected() == 1\n", + ) + (root / "tests/__init__.py").write_text("") + (root / "tests/helper.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "tests/helper.py" in executions[0].detail + + +def test_validator_subprocess_cannot_read_signing_secret(tmp_path, monkeypatch) -> None: + content = ( + "import os\nfrom pathlib import Path\n" + "def test_widget():\n" + " Path('observed-secret').write_text(" + "os.environ.get('PDD_ATTESTATION_SIGNING_KEY', 'missing'))\n" + ) + root, head = _repository(tmp_path, content) + monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-must-not-read") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + assert (root / "observed-secret").read_text() == "missing" + + +def test_ambient_pytest_options_and_plugins_are_disabled(tmp_path, monkeypatch) -> None: + content = ( + "import os\n" + "def test_widget():\n" + " assert os.environ.get('PYTEST_ADDOPTS') is None\n" + " assert os.environ['PYTEST_DISABLE_PLUGIN_AUTOLOAD'] == '1'\n" + ) + root, head = _repository(tmp_path, content) + monkeypatch.setenv("PYTEST_ADDOPTS", "--collect-only") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + + +def test_deselected_declared_test_cannot_pass(tmp_path) -> None: + content = "def test_keep(): assert True\ndef test_drop(): assert True\n" + root, head = _repository(tmp_path, content) + (root / "conftest.py").write_text( + "def pytest_collection_modifyitems(items):\n" + " items[:] = [item for item in items if item.name == 'test_keep']\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "protected collection policy") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.NOT_COLLECTED + + +def test_removed_parametrized_case_cannot_pass(tmp_path) -> None: + content = ( + "import pytest\n" + "@pytest.mark.parametrize('value', [1, 2])\n" + "def test_widget(value): assert value in {1, 2}\n" + ) + root, head = _repository(tmp_path, content) + (root / "conftest.py").write_text( + "def pytest_collection_modifyitems(items):\n" + " items[:] = [item for item in items if '[2]' not in item.nodeid]\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "protected collection hook") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_config_loaded_local_plugin_fails_closed(tmp_path) -> None: + root, head = _repository(tmp_path, "def test_widget(): assert False\n") + (root / "local_plugin.py").write_text( + "def pytest_collection_modifyitems(items):\n" + " for item in items: item.obj = lambda: None\n" + ) + (root / "pytest.ini").write_text("[pytest]\naddopts = -p local_plugin\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "configured local plugin") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "plugins are not bound" in executions[0].detail + + +def test_non_strict_xpass_cannot_pass(tmp_path) -> None: + content = ( + "import pytest\n" + "@pytest.mark.xfail(reason='known')\n" + "def test_widget(): assert True\n" + ) + root, head = _repository(tmp_path, content) + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.XFAIL diff --git a/tests/test_sync_core_snapshot.py b/tests/test_sync_core_snapshot.py new file mode 100644 index 000000000..67c6791d4 --- /dev/null +++ b/tests/test_sync_core_snapshot.py @@ -0,0 +1,105 @@ +"""Tests for complete multi-artifact snapshots and include closure binding.""" + +import json +import os +import subprocess +from pathlib import Path + +from pdd.sync_core import ( + build_unit_manifest, + build_unit_snapshot, + load_verification_profiles, +) +from pdd.sync_core.identity import initialize_repository_identity + + +REPOSITORY_ID = "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _repository(tmp_path: Path, *, query: bool = False) -> tuple[Path, str]: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "snapshot@example.com") + _git(root, "config", "user.name", "Snapshot Test") + initialize_repository_identity(root, REPOSITORY_ID) + for directory in ("prompts", "src", "docs", "tests"): + (root / directory).mkdir() + include = ( + 'docs/widget.md' + if query + else "docs/widget.md" + ) + (root / "prompts/widget_python.prompt").write_text(f"REQ-1: widget\n{include}\n") + (root / "docs/widget.md").write_text("Widget contract\n") + (root / "src/widget.py").write_text("value = 1\n") + (root / "tests/test_widget.py").write_text("def test_widget(): pass\n") + (root / "tests/test_widget_e2e.py").write_text("def test_e2e(): pass\n") + os.chmod(root / "tests/test_widget_e2e.py", 0o755) + (root / "architecture.json").write_text( + json.dumps( + [{"filename": "widget_python.prompt", "filepath": "src/widget.py"}] + ) + ) + (root / ".pdd/verification-profiles.json").write_text( + json.dumps( + { + "profiles": [ + { + "prompt_path": "prompts/widget_python.prompt", + "language_id": "python", + "required_requirement_ids": ["REQ-1"], + "obligations": [ + { + "obligation_id": "tests", + "kind": "test", + "validator_id": "pytest", + "validator_config_digest": "pytest-v1", + "requirement_ids": ["REQ-1"], + "artifact_paths": [ + "tests/test_widget.py", + "tests/test_widget_e2e.py", + ], + } + ], + } + ] + } + ) + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "snapshot fixture") + return root, _git(root, "rev-parse", "HEAD") + + +def test_snapshot_contains_prompt_include_code_and_all_tests(tmp_path) -> None: + root, commit = _repository(tmp_path) + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + profile = load_verification_profiles(root, manifest).profiles[0] + snapshot = build_unit_snapshot(root, manifest, manifest.managed_units[0], profile) + identities = {(item.role, item.relpath.as_posix()) for item in snapshot.artifacts} + assert identities == { + ("prompt", "prompts/widget_python.prompt"), + ("include", "docs/widget.md"), + ("code", "src/widget.py"), + ("test", "tests/test_widget.py"), + ("test", "tests/test_widget_e2e.py"), + } + executable = next( + item for item in snapshot.artifacts if item.relpath.name == "test_widget_e2e.py" + ) + assert executable.git_mode == "100755" + + +def test_query_expansion_cannot_receive_trusted_verified_status(tmp_path) -> None: + root, commit = _repository(tmp_path, query=True) + manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) + profile = load_verification_profiles(root, manifest).profiles[0] + snapshot = build_unit_snapshot(root, manifest, manifest.managed_units[0], profile) + assert snapshot.nondeterministic_inputs is True diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py new file mode 100644 index 000000000..031596686 --- /dev/null +++ b/tests/test_sync_core_transaction.py @@ -0,0 +1,181 @@ +"""Crash, CAS, mode, and recovery tests for the durable transaction WAL.""" + +import json +import os +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + PlannedWrite, + TransactionConflict, + TransactionError, + TransactionManager, + TransactionPhase, +) + + +def _writes(): + return ( + PlannedWrite(PurePosixPath("src/widget.py"), b"value = 2\n", "100755"), + PlannedWrite(PurePosixPath(".pdd/evidence/widget.json"), b"{}\n", "100644"), + PlannedWrite(PurePosixPath(".pdd/meta/v2/widget.json"), b"{}\n", "100644"), + ) + + +def test_success_commits_artifact_evidence_and_fingerprint_together(tmp_path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src/widget.py").write_text("value = 1\n") + manager = TransactionManager(tmp_path) + prepared = manager.prepare("tx-success", _writes()) + assert prepared.phase is TransactionPhase.PREPARED + assert (tmp_path / "src/widget.py").read_text() == "value = 1\n" + result = manager.commit("tx-success") + assert result.phase is TransactionPhase.COMMITTED + assert (tmp_path / "src/widget.py").read_text() == "value = 2\n" + assert (tmp_path / "src/widget.py").stat().st_mode & 0o777 == 0o755 + assert (tmp_path / ".pdd/evidence/widget.json").exists() + assert (tmp_path / ".pdd/meta/v2/widget.json").exists() + assert manager.incomplete() == () + + +def test_identical_plan_is_a_no_op_without_journal(tmp_path) -> None: + for write in _writes(): + path = tmp_path.joinpath(*write.relpath.parts) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(write.content) + os.chmod(path, 0o755 if write.git_mode == "100755" else 0o644) + result = TransactionManager(tmp_path).prepare("tx-noop", _writes()) + assert result.no_op is True + assert not (tmp_path / ".pdd/transactions/tx-noop").exists() + + +def test_external_edit_after_prepare_is_conflict_without_writes(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-conflict", _writes()) + target.write_text("external = True\n") + with pytest.raises(TransactionConflict, match="destination changed"): + manager.commit("tx-conflict") + assert target.read_text() == "external = True\n" + assert not (tmp_path / ".pdd/evidence/widget.json").exists() + + +@pytest.mark.parametrize("crash_event", ["after_committing", "after_install:0", "after_install:1"]) +def test_process_death_recovers_committing_transaction(tmp_path, crash_event) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src/widget.py").write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-crash", _writes()) + + def crash(event): + if event == crash_event: + raise SystemExit("simulated process death") + + with pytest.raises(SystemExit): + manager.commit("tx-crash", crash_hook=crash) + assert manager.incomplete() == ("tx-crash",) + recovered = manager.recover("tx-crash") + assert recovered.phase is TransactionPhase.COMMITTED + assert (tmp_path / "src/widget.py").read_text() == "value = 2\n" + assert (tmp_path / ".pdd/evidence/widget.json").exists() + assert (tmp_path / ".pdd/meta/v2/widget.json").exists() + assert manager.recover("tx-crash").no_op is True + + +def test_prepared_transaction_recovers_by_rollback_without_destination_write(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-prepared", _writes()) + assert manager.recover("tx-prepared").phase is TransactionPhase.ROLLED_BACK + assert target.read_text() == "value = 1\n" + assert manager.incomplete() == () + + +def test_corrupt_prepared_blob_never_commits(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-corrupt", _writes()) + blob = tmp_path / ".pdd/transactions/tx-corrupt/prepared-0.blob" + blob.write_bytes(b"attacker bytes\n") + with pytest.raises(TransactionError, match="prepared transaction blob is corrupt"): + manager.commit("tx-corrupt") + assert target.read_text() == "value = 1\n" + + +def test_later_corrupt_blob_is_detected_before_first_install(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-later-corrupt", _writes()) + transaction = tmp_path / ".pdd/transactions/tx-later-corrupt" + (transaction / "prepared-1.blob").write_bytes(b"attacker bytes\n") + with pytest.raises(TransactionError, match="prepared transaction blob is corrupt"): + manager.commit("tx-later-corrupt") + assert target.read_text() == "value = 1\n" + assert not (tmp_path / ".pdd/evidence/widget.json").exists() + + +def test_recovery_rolls_back_partial_install_when_later_blob_is_corrupt(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-partial-corrupt", _writes()) + + def crash(event): + if event == "after_install:0": + raise SystemExit("simulated process death") + + with pytest.raises(SystemExit): + manager.commit("tx-partial-corrupt", crash_hook=crash) + transaction = tmp_path / ".pdd/transactions/tx-partial-corrupt" + (transaction / "prepared-1.blob").write_bytes(b"attacker bytes\n") + with pytest.raises(TransactionError, match="prepared transaction blob is corrupt"): + manager.recover("tx-partial-corrupt") + assert target.read_text() == "value = 1\n" + assert not (tmp_path / ".pdd/evidence/widget.json").exists() + journal = json.loads((transaction / "journal.json").read_text()) + assert journal["phase"] == TransactionPhase.ROLLED_BACK.value + + +def test_read_only_incomplete_scan_does_not_change_journal_or_destinations(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-readonly", _writes()) + journal = tmp_path / ".pdd/transactions/tx-readonly/journal.json" + before = journal.read_bytes() + assert manager.incomplete() == ("tx-readonly",) + assert journal.read_bytes() == before + assert target.read_text() == "value = 1\n" + + +def test_journal_and_blobs_are_private_and_contain_modes(tmp_path) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src/widget.py").write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-private", _writes()) + transaction_dir = tmp_path / ".pdd/transactions/tx-private" + assert transaction_dir.stat().st_mode & 0o777 == 0o700 + for path in transaction_dir.iterdir(): + assert path.stat().st_mode & 0o777 == 0o600 + payload = json.loads((transaction_dir / "journal.json").read_text()) + assert payload["entries"][0]["desired_mode"] == "100755" + assert payload["entries"][0]["precondition"]["git_mode"] == "100644" + + +def test_secret_labeled_write_refuses_unencrypted_rollback(tmp_path) -> None: + secret = PlannedWrite( + PurePosixPath("secrets/token"), b"sensitive", "100644", secret=True + ) + with pytest.raises(TransactionError, match="requires configured encryption"): + TransactionManager(tmp_path).prepare("tx-secret", (secret,)) diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py new file mode 100644 index 000000000..f83140a71 --- /dev/null +++ b/tests/test_sync_core_trust.py @@ -0,0 +1,175 @@ +"""Adversarial tests for trusted semantic evidence issuance.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from pathlib import PurePosixPath + +import pytest + +from pdd.sync_core import ( + AttestationError, + AttestationBinding, + AttestationRequest, + AttestationSigner, + AttestationTrustPolicy, + EvidenceOutcome, + FileReplayStore, + UnitId, +) +from pdd.sync_core.trust import ValidationEvidence +from pdd.sync_core.types import ObligationEvidence + + +PRIVATE_KEY = b"t" * 32 +SIGNER = AttestationSigner("trusted-ci", PRIVATE_KEY) +PUBLIC_KEY = SIGNER.public_key_bytes() +NOW = datetime(2026, 7, 10, 12, 0, tzinfo=timezone.utc) +UNIT = UnitId("repository-1", PurePosixPath("prompts/widget_python.prompt"), "python") + + +def _envelope(*, nonce="nonce-1", issued_at=NOW, lifetime=timedelta(hours=1)): + return SIGNER.issue( + AttestationRequest( + "attestation-1", + _binding(), + (ObligationEvidence("test", EvidenceOutcome.PASS),), + nonce, + issued_at, + lifetime, + ) + ) + + +def _binding(**overrides): + values = { + "subject": UNIT, + "snapshot_digest": "snapshot-1", + "profile_digest": "profile-1", + "runner_digest": "runner-1", + "tool_version": "pdd-test", + "base_sha": "base-1", + "checked_sha": "head-1", + } + values.update(overrides) + return AttestationBinding(**values) + + +def _verify(policy, envelope, **overrides): + now = overrides.pop("now", NOW) + return policy.verify(envelope, _binding(**overrides), now=now) + + +def test_valid_attestation_produces_sealed_evidence() -> None: + evidence = _verify(AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}), _envelope()) + assert isinstance(evidence, ValidationEvidence) + assert evidence.attestation_id == "attestation-1" + + +def test_evidence_cannot_be_caller_asserted() -> None: + with pytest.raises(TypeError, match="AttestationTrustPolicy"): + ValidationEvidence( + object(), + _seal=object(), + ) + + +def test_forged_signature_is_rejected() -> None: + envelope = replace(_envelope(), binding=_binding(checked_sha="candidate-head")) + with pytest.raises(AttestationError, match="signature"): + _verify(AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}), envelope) + + +def test_unknown_issuer_is_rejected() -> None: + with pytest.raises(AttestationError, match="not trusted"): + _verify(AttestationTrustPolicy({}), _envelope()) + + +def test_expired_attestation_is_rejected() -> None: + envelope = _envelope(issued_at=NOW - timedelta(hours=2), lifetime=timedelta(minutes=1)) + with pytest.raises(AttestationError, match="expired"): + _verify(AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}), envelope) + + +def test_overlong_attestation_lifetime_is_rejected() -> None: + envelope = _envelope(nonce="nonce-long", lifetime=timedelta(hours=2)) + with pytest.raises(AttestationError, match="exceeds policy"): + _verify(AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}), envelope) + + +@pytest.mark.parametrize( + ("policy", "message"), + [ + (AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}, revoked_issuers=frozenset({"trusted-ci"})), "issuer is revoked"), + (AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}, revoked_attestations=frozenset({"attestation-1"})), "attestation is revoked"), + ], +) +def test_revocation_is_enforced(policy, message) -> None: + with pytest.raises(AttestationError, match=message): + _verify(policy, _envelope()) + + +def test_nonce_reuse_by_different_attestation_is_rejected() -> None: + policy = AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}) + first = _envelope() + _verify(policy, first) + second = SIGNER.issue( + AttestationRequest( + "attestation-2", + _binding(), + first.results, + "nonce-1", + NOW, + ) + ) + with pytest.raises(AttestationError, match="replayed"): + _verify(policy, second) + + +def test_exact_signed_statement_replay_is_rejected() -> None: + policy = AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}) + envelope = _envelope() + _verify(policy, envelope) + with pytest.raises(AttestationError, match="replayed"): + _verify(policy, envelope) + + +def test_durable_nonce_collision_is_rejected_across_policy_instances(tmp_path) -> None: + path = tmp_path / "external/replay.json" + envelope = _envelope() + first = AttestationTrustPolicy( + {"trusted-ci": PUBLIC_KEY}, replay_store=FileReplayStore(path) + ) + second = AttestationTrustPolicy( + {"trusted-ci": PUBLIC_KEY}, replay_store=FileReplayStore(path) + ) + _verify(first, envelope) + conflicting = SIGNER.issue( + AttestationRequest( + "attestation-conflict", + _binding(), + envelope.results, + "nonce-1", + NOW, + ) + ) + with pytest.raises(AttestationError, match="replayed"): + _verify(second, conflicting) + assert path.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize( + "override", + [ + {"snapshot_digest": "snapshot-2"}, + {"profile_digest": "profile-2"}, + {"base_sha": "base-2"}, + {"checked_sha": "head-2"}, + ], +) +def test_wrong_closure_binding_is_rejected(override) -> None: + with pytest.raises(AttestationError, match="checked closure"): + _verify( + AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}), + _envelope(), + **override, + ) diff --git a/tests/test_sync_core_verification_profiles.py b/tests/test_sync_core_verification_profiles.py new file mode 100644 index 000000000..2e5119cd5 --- /dev/null +++ b/tests/test_sync_core_verification_profiles.py @@ -0,0 +1,161 @@ +"""Tests for protected base/head verification-profile authority.""" + +import json +import hashlib +import subprocess +from pathlib import Path + +from pdd.sync_core import build_unit_manifest, load_verification_profiles +from pdd.sync_core.identity import initialize_repository_identity + + +REPOSITORY_ID = "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=True + ).stdout.strip() + + +def _commit(root: Path, message: str) -> str: + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", message) + return _git(root, "rev-parse", "HEAD") + + +def _profile(requirements=None, obligations=None): + return { + "profiles": [ + { + "prompt_path": "prompts/widget_python.prompt", + "language_id": "python", + "required_requirement_ids": ( + ["REQ-1"] if requirements is None else requirements + ), + "obligations": ( + [ + { + "obligation_id": "pytest", + "kind": "test", + "validator_id": "pytest", + "validator_config_digest": "pytest-v1", + "requirement_ids": ["REQ-1"], + "artifact_paths": ["tests/test_widget.py"], + "required": True, + } + ] + if obligations is None + else obligations + ), + } + ] + } + + +def _repository(tmp_path: Path) -> Path: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "profiles@example.com") + _git(root, "config", "user.name", "Profiles Test") + initialize_repository_identity(root, REPOSITORY_ID) + (root / "prompts").mkdir() + (root / "prompts/widget_python.prompt").write_text("REQ-1: Build widget\n") + return root + + +def _manifest(root: Path, base: str, head: str): + return build_unit_manifest(root, base_ref=base, head_ref=head) + + +def test_complete_protected_profile_has_full_coverage(tmp_path) -> None: + root = _repository(tmp_path) + (root / ".pdd/verification-profiles.json").write_text(json.dumps(_profile())) + commit = _commit(root, "profile") + profiles = load_verification_profiles(root, _manifest(root, commit, commit)) + assert profiles.coverage == 1.0 + assert profiles.invalid_reasons == () + + +def test_missing_profile_is_explicit_and_incomplete(tmp_path) -> None: + root = _repository(tmp_path) + commit = _commit(root, "no profile") + profiles = load_verification_profiles(root, _manifest(root, commit, commit)) + assert profiles.coverage == 0.0 + assert any("profile is missing" in item for item in profiles.invalid_reasons) + assert profiles.profiles[0].complete is False + + +def test_candidate_cannot_delete_protected_obligation(tmp_path) -> None: + root = _repository(tmp_path) + profile_path = root / ".pdd/verification-profiles.json" + profile_path.write_text(json.dumps(_profile())) + base = _commit(root, "base profile") + profile_path.write_text(json.dumps(_profile(obligations=[]))) + head = _commit(root, "delete obligation") + profiles = load_verification_profiles(root, _manifest(root, base, head)) + effective = profiles.profiles[0] + assert [item.obligation_id for item in effective.obligations] == ["pytest"] + assert any("removed protected obligation" in item for item in profiles.invalid_reasons) + + +def test_candidate_cannot_remap_protected_validator(tmp_path) -> None: + root = _repository(tmp_path) + profile_path = root / ".pdd/verification-profiles.json" + profile_path.write_text(json.dumps(_profile())) + base = _commit(root, "base profile") + changed = _profile() + changed["profiles"][0]["obligations"][0]["validator_id"] = "candidate-validator" + profile_path.write_text(json.dumps(changed)) + head = _commit(root, "remap validator") + profiles = load_verification_profiles(root, _manifest(root, base, head)) + assert profiles.profiles[0].obligations[0].validator_id == "pytest" + assert any("changed protected obligation" in item for item in profiles.invalid_reasons) + + +def test_new_requirement_without_mapping_is_incomplete(tmp_path) -> None: + root = _repository(tmp_path) + profile_path = root / ".pdd/verification-profiles.json" + profile_path.write_text(json.dumps(_profile())) + base = _commit(root, "base profile") + (root / "prompts/widget_python.prompt").write_text( + "REQ-1: Build widget\nREQ-2: Reject invalid input\n" + ) + profile_path.write_text(json.dumps(_profile(requirements=["REQ-1", "REQ-2"]))) + head = _commit(root, "new unmapped requirement") + profiles = load_verification_profiles(root, _manifest(root, base, head)) + assert profiles.coverage == 0.0 + assert any("profile is incomplete" in item for item in profiles.invalid_reasons) + + +def test_profile_cannot_invent_smaller_requirement_universe(tmp_path) -> None: + root = _repository(tmp_path) + (root / "prompts/widget_python.prompt").write_text( + "REQ-1: Build widget\nREQ-2: Reject invalid input\n" + ) + profile_path = root / ".pdd/verification-profiles.json" + profile_path.write_text(json.dumps(_profile(requirements=["REQ-1"]))) + commit = _commit(root, "dishonest profile") + profiles = load_verification_profiles(root, _manifest(root, commit, commit)) + assert any( + "do not match immutable prompt requirements" in item + for item in profiles.invalid_reasons + ) + assert profiles.coverage == 0.0 + + +def test_prompt_without_explicit_ids_uses_full_contract_digest(tmp_path) -> None: + root = _repository(tmp_path) + prompt = root / "prompts/widget_python.prompt" + prompt.write_text("Build a widget with validated input.\n") + digest = hashlib.sha256(prompt.read_bytes()).hexdigest() + profile = _profile(requirements=[f"CONTRACT-SHA256:{digest}"]) + profile["profiles"][0]["obligations"][0]["requirement_ids"] = [ + f"CONTRACT-SHA256:{digest}" + ] + (root / ".pdd/verification-profiles.json").write_text(json.dumps(profile)) + commit = _commit(root, "contract digest") + profiles = load_verification_profiles(root, _manifest(root, commit, commit)) + assert profiles.invalid_reasons == () + assert profiles.coverage == 1.0 From 0b6118594601cc47de48435c5472a96acad3a2f1 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 16:52:39 -0700 Subject: [PATCH 002/128] feat: enforce canonical sync proof boundaries --- docs/global_sync_resolution_plan.md | 13 +- pdd/auto_deps_main.py | 9 + pdd/ci_drift_heal.py | 58 ++++- pdd/commands/reconcile.py | 9 +- pdd/commands/sync_core.py | 69 ++++-- pdd/construct_paths.py | 46 +--- pdd/continuous_sync.py | 191 +++++++++++++--- pdd/data/language_format.csv | 1 + pdd/get_language.py | 35 +-- pdd/operation_log.py | 9 + pdd/pin_example_hack.py | 10 +- pdd/preprocess.py | 45 ++-- pdd/sync_core/__init__.py | 8 + pdd/sync_core/certificate.py | 135 ++++++++++-- pdd/sync_core/decommission.py | 102 +++++++++ pdd/sync_core/finalize.py | 58 +++-- pdd/sync_core/git_io.py | 14 ++ pdd/sync_core/isolation.py | 10 + pdd/sync_core/lifecycle.py | 68 ++++-- pdd/sync_core/manifest.py | 185 ++++++++++------ pdd/sync_core/reporting.py | 18 +- pdd/sync_core/runner.py | 108 +++++++--- pdd/sync_core/scenario_contract.py | 2 + pdd/sync_core/scenario_harness.py | 203 +++++++++++++++++- pdd/sync_core/snapshot.py | 2 +- pdd/sync_core/transaction.py | 16 ++ pdd/sync_core/trust.py | 98 +++++++-- pdd/sync_core/types.py | 13 +- pdd/sync_core/verification.py | 5 + pdd/sync_determine_operation.py | 152 ++++++------- pdd/sync_orchestration.py | 58 ++++- pdd/sync_order.py | 67 +----- pdd/update_main.py | 8 +- ...st_issue_1932_continuous_sync_guarantee.py | 90 ++++---- tests/test_ci_drift_heal.py | 17 +- tests/test_get_language.py | 23 +- tests/test_preprocess.py | 19 +- tests/test_sync_core_certificate.py | 91 +++++++- tests/test_sync_core_manifest.py | 58 ++++- tests/test_sync_core_reporting.py | 17 +- tests/test_sync_core_runner.py | 14 +- tests/test_sync_core_snapshot.py | 40 +++- tests/test_sync_core_transaction.py | 17 ++ tests/test_sync_core_trust.py | 77 ++++++- tests/test_sync_core_verification_profiles.py | 16 +- 45 files changed, 1676 insertions(+), 628 deletions(-) create mode 100644 pdd/sync_core/decommission.py create mode 100644 pdd/sync_core/isolation.py diff --git a/docs/global_sync_resolution_plan.md b/docs/global_sync_resolution_plan.md index 533821514..86ccceb26 100644 --- a/docs/global_sync_resolution_plan.md +++ b/docs/global_sync_resolution_plan.md @@ -70,14 +70,15 @@ of Done or authorize a partial green certificate. | Workstream | Current evidence | State | | --- | --- | --- | | Canonical core | Identity, immutable Git manifest, include closure, hashing, classifier, profiles, trust, reporting, transaction WAL, and compatibility routing implemented under `pdd/sync_core/` | implemented, not rolled out | -| Exact certification | Independent exact-SHA clones, clean-tree/ancestry checks, strict fresh green certificate verification, exact issuer and repository refs | implemented and unit tested | -| Trusted test runner | Strict pytest invocation; ambient options/plugins disabled; config, `conftest.py`, and transitive local helper closure signed; protected and candidate node IDs compared before each protected node executes independently | implemented for pytest only; non-pytest adapters remain | -| Temporal proof | Signed rows require complete predicates, repository identity, SHA ancestry, consecutive UTC dates, and no-write lifecycle metrics | implemented; seven real nights absent | -| Lifecycle harness | A credential-free checker-owned packaged harness replaces candidate pytest files; all required scenario IDs execute from an isolated built wheel | implemented locally; protected release/workflow deployment remains | -| PDD verification | `278 passed` for `tests/test_sync_core_*.py`; checker-owned scenarios also execute from an isolated built wheel | component green, E2E red | +| Exact certification | Independent exact-SHA clones, clean-tree/ancestry checks, strict fresh green certificate verification, exact issuer/repository refs, and remote-signature verification without local private keys | implemented and unit tested; protected signer service absent | +| Trusted test runner | Strict pytest invocation; ambient options/plugins disabled; config, `conftest.py`, and transitive local helper closure signed; protected and candidate node IDs compared before each protected node executes independently; candidate-only profiles rejected and actual pytest config digest checked | implemented for pytest only; non-pytest and threshold-human adapters remain | +| Temporal proof | Signed rows require complete predicates, repository identity, SHA ancestry, consecutive UTC dates, deleted-ledger full scans, normal no-op observations, injected canary outcomes, and zero-write reruns; exact immutable attestation rechecks are idempotent while nonce rebinding fails | implemented locally; protected timestamped storage and seven real nights absent | +| Lifecycle harness | A released checker installs the exact candidate wheel in a separate credential-free environment, drives public candidate report/recovery commands, and independently runs the required scenario registry and real cloud canary | split-install wheel run: one expected cloud-canary failure, zero skips/errors/timeouts/no-op writes/tree changes/missing scenarios; protected release/workflow deployment remains | +| PDD verification | `290 passed` for `tests/test_sync_core_*.py` with no skips; broad compatibility suite reached 1,145 pass/4 coverage-capture failures and all four focused regressions pass after the activation fast-path repair; unit snapshots no longer drift on unrelated human-owned changes | component green, E2E red | | pdd_cloud boundary | Exact-tree path/Python/prompt/architecture scan reports zero forbidden vendored semantics at `677a9c88f`; the legacy finalizer declarations and implementation are removed and current consumer installs are exact-version pinned | implemented; final pin must target the protected reviewed release | -| pdd_cloud inventory | Exact `HEAD^..HEAD` report at `677a9c88f`: 572 current managed, 573 protected expected, 534 unbaselined/unknown, 38 drifted/failed, one removed-unit invalidity, zero profiles/evidence, and zero unaccounted tracked paths | two-phase decommission plus profiles/evidence migration remain | +| pdd_cloud inventory | Protected registry/tombstone commit `39b78e487` retains 573 historical identities and authorizes only the retired finalizer; both transition and stable manifests report 572 managed = 572 expected, zero invalid, and zero unaccounted | profiles/evidence migration remains | | Global certificate | Scan remains red for protected checker deployment, transactional staging, profile/evidence debt, and nightly history | correctly blocked | +| Adversarial review | xhigh round 8 returned `NOT APPROVED`; unit-global manifest coupling, candidate-wheel coverage, profile authority, replay/nightly observations, protected denominator, rollback preflight, and local signing-key findings have been addressed locally | generation staging/no-follow, release provenance, semantic ownership audit, adapter matrix, rollout, and seven nights remain | No acceptance claim is valid until the signed certificate recomputes `ok: true` for exact protected PDD and pdd_cloud SHAs and a verifier supplied with the diff --git a/pdd/auto_deps_main.py b/pdd/auto_deps_main.py index 101697c4c..94a96a48f 100644 --- a/pdd/auto_deps_main.py +++ b/pdd/auto_deps_main.py @@ -254,12 +254,18 @@ def auto_deps_main( model=model_name, ) except Exception as fp_exc: + from .sync_core.finalize import CanonicalFinalizationError + if isinstance(fp_exc, CanonicalFinalizationError): + raise if not quiet: console.print( f"[yellow]Warning: Failed to save fingerprint for " f"{basename}_{language}: {fp_exc}[/yellow]" ) except Exception as meta_exc: + from .sync_core.finalize import CanonicalFinalizationError + if isinstance(meta_exc, CanonicalFinalizationError): + raise # Never mask a successful auto-deps result on metadata errors if not quiet: console.print( @@ -272,6 +278,9 @@ def auto_deps_main( # Re-raise to allow orchestrators (e.g. pdd sync) to stop the loop raise except Exception as exc: + from .sync_core.finalize import CanonicalFinalizationError + if isinstance(exc, CanonicalFinalizationError): + raise if not quiet: console.print(f"[red]Error in auto-deps: {exc}[/red]") return "", 0.0, f"Error: {exc}" diff --git a/pdd/ci_drift_heal.py b/pdd/ci_drift_heal.py index a87dd695d..af8b675f0 100644 --- a/pdd/ci_drift_heal.py +++ b/pdd/ci_drift_heal.py @@ -11,6 +11,7 @@ import argparse import csv +import json import math import os import re @@ -299,25 +300,29 @@ def _discover_modules() -> List[Tuple[str, str, Any]]: """Discover (basename, language, prompt_path) tuples.""" try: from pdd.user_story_tests import discover_prompt_files - except ImportError: - return [] + except ImportError as exc: + raise RuntimeError("cannot import prompt discovery") from exc try: prompt_files = discover_prompt_files() - except Exception: - return [] + except Exception as exc: + raise RuntimeError("prompt discovery failed") from exc try: from pdd.operation_log import infer_module_identity - except ImportError: - return [] + except ImportError as exc: + raise RuntimeError("cannot import canonical module identity") from exc out: List[Tuple[str, str, Any]] = [] + failures: List[str] = [] for entry in prompt_files: try: basename, language = infer_module_identity(str(entry)) - except Exception: + except Exception as exc: + failures.append(f"{entry}: {exc}") continue out.append((basename, language, entry)) + if failures: + raise RuntimeError("module identity failed: " + "; ".join(failures)) return out @@ -412,6 +417,8 @@ def detect_drift( changed_files = _get_git_changed_files(diff_base) discovered = _discover_modules() + if not discovered and parsed is None: + raise RuntimeError("no synchronization units were discovered") if parsed is not None: wanted = set(parsed) discovered = [t for t in discovered if t[0] in wanted] @@ -459,8 +466,10 @@ def detect_drift( decision = sync_determine_operation( basename, language, target_coverage=90.0, log_mode=True ) - except Exception: - continue + except Exception as exc: + raise RuntimeError( + f"classification failed for {basename}/{language}: {exc}" + ) from exc op = _extract_op(decision) if not op or op in ("nothing", "synced"): @@ -1848,6 +1857,37 @@ def main( as_json: bool = False, ) -> int: """Detect drift, heal modules, and commit healed changes.""" + from pdd.continuous_sync import canonical_sync_enabled + + if canonical_sync_enabled(Path.cwd()) and not dry_run: + from pdd.sync_core import CanonicalReportOptions, build_canonical_report + + base_ref = ( + diff_base.split("...", 1)[0] + if diff_base and "..." in diff_base + else "HEAD" + ) + try: + canonical = build_canonical_report( + _repo_root(), + CanonicalReportOptions( + base_ref=base_ref, + head_ref="HEAD", + modules=tuple(modules or ()), + ), + ) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]canonical sync check failed: {exc}[/red]") + return 1 + if not canonical["ok"]: + if as_json: + print(json.dumps(canonical, indent=2, sort_keys=True)) + else: + console.print("[red]canonical sync predicate failed[/red]") + for error in canonical.get("errors", []): + console.print(f"[red]- {error}[/red]") + return 1 + return 0 if dry_run: from pdd.continuous_sync import build_report import json as _json diff --git a/pdd/commands/reconcile.py b/pdd/commands/reconcile.py index 0eb95ff6d..341a88ab5 100644 --- a/pdd/commands/reconcile.py +++ b/pdd/commands/reconcile.py @@ -75,7 +75,7 @@ def _emit_report(report: dict, as_json: bool) -> None: "--heal", is_flag=True, default=False, - help="Refresh valid stale fingerprints without LLM calls.", + help="Deprecated; blind fingerprint acceptance is disabled.", ) @click.option( "--ledger", @@ -103,10 +103,15 @@ def reconcile( ctx.ensure_object(dict) if as_json: ctx.obj["_suppress_result_summary"] = True + if heal: + raise click.UsageError( + "--heal is disabled because changing a fingerprint does not prove " + "semantic synchronization; run pdd sync/update/resolve instead" + ) report = build_report( consumer="reconcile", modules=[module_name] if module_name else None, - heal=heal, + heal=False, ledger=ledger, ) _emit_report(report, as_json) diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py index b37c2ca94..9a62750a7 100644 --- a/pdd/commands/sync_core.py +++ b/pdd/commands/sync_core.py @@ -4,7 +4,6 @@ import json import os -import subprocess import uuid from datetime import datetime, timezone from pathlib import Path @@ -19,6 +18,7 @@ FingerprintRecord, FingerprintStore, GlobalCertificateOptions, + NightlyObservation, PlannedWrite, RepositoryTarget, SemanticStatus, @@ -35,6 +35,7 @@ run_lifecycle_matrix, signer_from_environment, ) +from ..sync_core.git_io import resolve_git_commit from .. import __version__ @@ -93,17 +94,48 @@ def _global_targets( return tuple(targets) -def _head_sha(root: Path) -> str: - result = subprocess.run( - ["git", "rev-parse", "--verify", "HEAD^{commit}"], - cwd=root, - capture_output=True, - text=True, - check=False, +def _load_nightly_observation(path: Path | None) -> NightlyObservation | None: + """Load protected workflow observations without accepting schema drift.""" + if path is None: + return None + payload = json.loads(path.read_text(encoding="utf-8")) + required = { + "complete_scan", + "ledgers_deleted_before_scan", + "normal_scan_writes", + "injected_canary_detected", + "injected_canary_outcome", + "post_canary_rerun_writes", + } + if not isinstance(payload, dict) or set(payload) != required: + raise ValueError("nightly observation schema is invalid") + if ( + not all( + isinstance(payload[name], bool) + for name in ( + "complete_scan", + "ledgers_deleted_before_scan", + "injected_canary_detected", + ) + ) + or not all( + isinstance(payload[name], int) + and not isinstance(payload[name], bool) + and payload[name] >= 0 + for name in ("normal_scan_writes", "post_canary_rerun_writes") + ) + or payload["injected_canary_outcome"] not in {"REPAIRED", "BLOCKED"} + ): + raise ValueError("nightly observation types are invalid") + observation = NightlyObservation( + payload["complete_scan"], + payload["ledgers_deleted_before_scan"], + payload["normal_scan_writes"], + payload["injected_canary_detected"], + payload["injected_canary_outcome"], + payload["post_canary_rerun_writes"], ) - if result.returncode != 0 or not result.stdout.strip(): - raise ValueError("cannot resolve repository HEAD") - return result.stdout.strip() + return observation @click.command("certify") @@ -125,6 +157,11 @@ def _head_sha(root: Path) -> str: type=click.Path(path_type=Path, dir_okay=False), envvar="PDD_NIGHTLY_CERTIFICATE_LEDGER", ) +@click.option( + "--nightly-observation", + type=click.Path(path_type=Path, dir_okay=False), + envvar="PDD_NIGHTLY_OBSERVATION", +) @click.option("--output", type=click.Path(path_type=Path, dir_okay=False)) @click.pass_context def certify( @@ -175,6 +212,11 @@ def certify( replay_ledger_root=replay_ledger, lifecycle_result=run_lifecycle_matrix( targets[0].path, + candidate_wheel=( + Path(os.environ["PDD_CANDIDATE_WHEEL"]) + if "PDD_CANDIDATE_WHEEL" in os.environ + else None + ), cloud_root=targets[1].path, cloud_base_ref=targets[1].base_ref, cloud_head_ref=targets[1].head_ref, @@ -186,6 +228,9 @@ def certify( ), required_nightly_streak=int(options["require_nightly_streak"]), checker_identity=checker_identity_from_environment(), + nightly_observation=_load_nightly_observation( + options.get("nightly_observation") + ), ), signer=signer_from_environment(), ) @@ -241,7 +286,7 @@ def baseline(ctx: click.Context, module: str, reviewed_by: str, reason: str) -> """Record reviewed current bytes as CURRENT plus semantic UNKNOWN.""" ctx.ensure_object(dict) root = Path.cwd().resolve() - head = _head_sha(root) + head = resolve_git_commit(root, "HEAD") manifest = build_unit_manifest(root, base_ref=head, head_ref=head) wanted = PurePosixPath(module) matches = [ diff --git a/pdd/construct_paths.py b/pdd/construct_paths.py index c48ab9ae9..cf1a29dfc 100644 --- a/pdd/construct_paths.py +++ b/pdd/construct_paths.py @@ -765,53 +765,19 @@ def _candidate_prompt_path(input_files: Dict[str, Path]) -> Path | None: def _is_known_language(language_name: str) -> bool: """Return True if the language is recognized. - Prefer CSV in PDD_PATH if available; otherwise fall back to a built-in set - so basename/language inference does not fail when PDD_PATH is unset. + Resolve only through the package-bundled protected language registry. """ language_name_lower = (language_name or "").lower() if not language_name_lower: return False - builtin_languages = { - 'python', 'javascript', 'typescript', 'typescriptreact', 'javascriptreact', - 'java', 'cpp', 'c', 'go', 'ruby', 'rust', - 'kotlin', 'swift', 'csharp', 'php', 'scala', 'r', 'lua', 'perl', 'bash', 'shell', - 'powershell', 'sql', 'prompt', 'html', 'css', 'makefile', - # Additional languages from language_format.csv - 'haskell', 'dart', 'elixir', 'clojure', 'julia', 'erlang', 'fortran', - 'nim', 'ocaml', 'groovy', 'coffeescript', 'fish', 'zsh', - 'prisma', 'lean', 'agda', - # Frontend / templating - 'svelte', 'vue', 'scss', 'sass', 'less', - 'jinja', 'handlebars', 'pug', 'ejs', 'twig', - # Modern / systems languages - 'zig', 'mojo', 'solidity', - # Config / query / infra - 'graphql', 'protobuf', 'terraform', 'hcl', 'nix', - 'glsl', 'wgsl', 'starlark', 'dockerfile', - # Common data and config formats for architecture prompts and configs - 'json', 'jsonl', 'yaml', 'yml', 'toml', 'ini' - } - - pdd_path_str = os.getenv('PDD_PATH') - if not pdd_path_str: - return language_name_lower in builtin_languages - - csv_file_path = Path(pdd_path_str) / 'data' / 'language_format.csv' - if not csv_file_path.is_file(): - return language_name_lower in builtin_languages + from pdd.sync_core.language import LanguageRegistry, LanguageRegistryError try: - with open(csv_file_path, mode='r', encoding='utf-8', newline='') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - if row.get('language', '').lower() == language_name_lower: - return True - except csv.Error as e: - console.print(f"[error]CSV Error reading {csv_file_path}: {e}", style="error") - return language_name_lower in builtin_languages - - return language_name_lower in builtin_languages + LanguageRegistry.bundled().resolve_alias(language_name_lower) + except LanguageRegistryError: + return False + return True # Languages that only produce a code output (no test/example). diff --git a/pdd/continuous_sync.py b/pdd/continuous_sync.py index a1ae64bae..7f0a60333 100644 --- a/pdd/continuous_sync.py +++ b/pdd/continuous_sync.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import os import subprocess from dataclasses import dataclass from datetime import datetime, timezone @@ -13,7 +14,6 @@ _safe_basename, get_fingerprint_path, infer_module_identity, - save_fingerprint, ) from .sync_determine_operation import ( Fingerprint, @@ -22,6 +22,7 @@ get_pdd_file_paths, read_fingerprint, ) +from .sync_core import CanonicalReportOptions, build_canonical_report DRIFT_CLASSIFICATIONS = { @@ -70,6 +71,62 @@ def project_root(start: Optional[Path] = None) -> Path: return current +def repository_root(start: Optional[Path] = None) -> Path: + """Return the enclosing Git repository root independently of `.pddrc`.""" + current = (start or Path.cwd()).resolve() + if not current.is_dir(): + current = current.parent + while not current.exists() and current.parent != current: + current = current.parent + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=current, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + return current + return Path(result.stdout.strip()).resolve() + + +def canonical_sync_enabled(root: Path) -> bool: + """Return whether protected committed policy activates canonical sync.""" + protected_ref = os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") + candidate = Path(root).resolve() + if not candidate.is_dir(): + candidate = candidate.parent + if protected_ref is None and not any( + (parent / ".pdd/sync-policy.json").is_file() + for parent in (candidate, *candidate.parents) + ): + return False + root = repository_root(candidate) + protected_ref = protected_ref or "HEAD" + identity = subprocess.run( + ["git", "cat-file", "-e", f"{protected_ref}:.pdd/repository-id"], + cwd=root, + capture_output=True, + check=False, + ) + if identity.returncode != 0: + return False + policy = subprocess.run( + ["git", "show", f"{protected_ref}:.pdd/sync-policy.json"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if policy.returncode != 0: + return False + try: + payload = json.loads(policy.stdout) + except json.JSONDecodeError: + return False + return payload == {"schema_version": 1, "enforcement": "active"} + + def _prompts_dir_for(prompt_path: Path) -> Path: """Return the concrete prompts directory to pass into path resolution.""" return prompt_path.parent @@ -281,9 +338,6 @@ def discover_units( units.append(unit) return units - if not metadata_identities: - return prompt_units - units: List[SyncUnit] = [] seen: set[tuple[str, str, Path]] = set() for identity in metadata_identities: @@ -293,6 +347,12 @@ def discover_units( continue seen.add(dedupe_key) units.append(unit) + for unit in prompt_units: + dedupe_key = (unit.basename, unit.language, unit.prompt_path) + if dedupe_key in seen: + continue + seen.add(dedupe_key) + units.append(unit) return units @@ -663,32 +723,6 @@ def _append_ledger( return str(ledger_path) if wrote else None -def _heal_units( - units: List[SyncUnit], - classified: List[Dict[str, Any]], -) -> List[str]: - healed: List[str] = [] - by_key = {(unit.basename, unit.language): unit for unit in units} - for item in classified: - if item["classification"] not in DRIFT_CLASSIFICATIONS: - continue - unit = by_key.get((item["basename"], item["language"])) - if unit is None: - continue - paths = get_pdd_file_paths( - unit.basename, - unit.language, - prompts_dir=str(unit.prompts_dir), - ) - fingerprint = read_fingerprint(unit.basename, unit.language, paths=paths) - operation = "fix" - if fingerprint and fingerprint.command in {"verify", "test", "fix", "update"}: - operation = fingerprint.command - save_fingerprint(unit.basename, unit.language, operation, paths, 0.0, "reconcile") - healed.append(unit.basename) - return healed - - def build_report( *, consumer: str, @@ -698,16 +732,25 @@ def build_report( ledger: bool = False, ) -> Dict[str, Any]: """Build a shared continuous-sync JSON report.""" + if heal: + raise ValueError( + "blind fingerprint healing is disabled; use an explicit repair or " + "reviewed baseline workflow" + ) base = project_root(root) + if canonical_sync_enabled(base): + if ledger: + raise ValueError( + "canonical read-only reporting cannot append a repository ledger" + ) + return _canonical_compatibility_report(base, consumer, modules) units = discover_units(base, modules=modules) classified = [classify_unit(unit, base) for unit in units] - healed = _heal_units(units, classified) if heal else [] - if healed: - classified = [classify_unit(unit, base) for unit in units] summary = _build_summary(classified) ledger_path = _append_ledger(base, consumer, classified) if ledger else None ok = ( - summary["metadata_stale"] == 0 + summary["total"] > 0 + and summary["metadata_stale"] == 0 and summary["conflicts"] == 0 and summary["unbaselined"] == 0 and summary["failures"] == 0 @@ -739,8 +782,86 @@ def build_report( for unit in classified if unit["classification"] == FAILURE_CLASSIFICATION ], - "healed": healed, + "healed": [], } if ledger_path: report["ledger_path"] = ledger_path return report + + +def _canonical_compatibility_report( + root: Path, + consumer: str, + modules: Optional[Iterable[str]], +) -> Dict[str, Any]: + """Project the trusted canonical report into the legacy consumer schema.""" + canonical = build_canonical_report( + root, + CanonicalReportOptions(modules=tuple(modules or ())), + ) + projected = [] + for unit in canonical["units"]: + baseline = unit["baseline"] + semantic = unit["semantic"] + if unit["in_sync"]: + classification = "IN_SYNC" + elif baseline == "UNBASELINED": + classification = UNBASELINED_CLASSIFICATION + elif semantic == "CONFLICT": + classification = CONFLICT_CLASSIFICATION + elif baseline == "DRIFTED": + classification = "DERIVED_CHANGED" + else: + classification = FAILURE_CLASSIFICATION + projected.append( + { + "basename": Path(unit["subject"]).stem.rsplit("_", 1)[0], + "language": Path(unit["subject"]).stem.rsplit("_", 1)[-1], + "classification": classification, + "operation": "none", + "reason": unit["reason"], + "changed_files": unit["changed_roles"], + "metadata_valid": baseline not in {"UNBASELINED", "CORRUPT"}, + "subject": unit["subject"], + } + ) + summary = _build_summary(projected) + counts = canonical["counts"] + summary["metadata_stale"] = counts["drifted"] + summary["conflicts"] = counts["conflict"] + summary["unbaselined"] = counts["unbaselined"] + summary["failures"] = ( + counts["corrupt"] + counts["unknown"] + counts["failed"] + counts["invalid"] + ) + summary["synced"] = counts["trusted_in_sync"] + summary["total"] = counts["managed_units"] + return { + "ok": canonical["ok"], + "consumer": consumer, + "project_root": str(root), + "summary": summary, + "metadata_stale": summary["metadata_stale"], + "units": projected, + "drift": [ + unit + for unit in projected + if unit["classification"] in DRIFT_CLASSIFICATIONS + ], + "conflicts": [ + unit + for unit in projected + if unit["classification"] == CONFLICT_CLASSIFICATION + ], + "unbaselined": [ + unit + for unit in projected + if unit["classification"] == UNBASELINED_CLASSIFICATION + ], + "failures": [ + unit + for unit in projected + if unit["classification"] == FAILURE_CLASSIFICATION + ], + "healed": [], + "canonical": canonical, + } diff --git a/pdd/data/language_format.csv b/pdd/data/language_format.csv index 1d7bcaf4e..81ceaf15b 100644 --- a/pdd/data/language_format.csv +++ b/pdd/data/language_format.csv @@ -1,6 +1,7 @@ language,comment,extension,run_command,run_test_command,outputs Python,#,.py,python {file},python -m pytest {file} -v,code|test|example Java,//,.java,java {file},,code|test|example +C,//,.c,,,code|test|example C++,//,.cpp,,,code|test|example JavaScript,//,.js,node {file},node {file},code|test|example HTML,"",.html,,,code diff --git a/pdd/get_language.py b/pdd/get_language.py index 974b4a4da..06ca937a9 100644 --- a/pdd/get_language.py +++ b/pdd/get_language.py @@ -1,5 +1,6 @@ -import csv -from pdd.path_resolution import get_default_resolver +"""Compatibility language lookup backed by the protected bundled registry.""" + +from pdd.sync_core import LanguageRegistry, LanguageRegistryError def get_language(extension: str) -> str: """ @@ -14,29 +15,9 @@ def get_language(extension: str) -> str: Raises: ValueError: If PDD_PATH environment variable is not set. """ - # Step 1: Resolve CSV path from PDD_PATH - resolver = get_default_resolver() + if not extension: + return "" try: - csv_path = resolver.resolve_data_file("data/language_format.csv") - except ValueError as exc: - raise ValueError("PDD_PATH environment variable is not set") from exc - - # Step 2: Ensure the extension starts with a dot and convert to lowercase - if not extension.startswith('.'): - extension = '.' + extension - extension = extension.lower() - - # Step 3 & 4: Look up the language name and handle exceptions - try: - with open(csv_path, 'r') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - if row['extension'].lower() == extension: - language = row['language'].strip() - return language if language else '' - except FileNotFoundError: - print(f"CSV file not found at {csv_path}") - except csv.Error as e: - print(f"Error reading CSV file: {e}") - - return '' # Return empty string if extension not found or any error occurs + return LanguageRegistry.bundled().resolve_extension(extension).display_name + except LanguageRegistryError: + return "" diff --git a/pdd/operation_log.py b/pdd/operation_log.py index 2bcf9f841..446aa7198 100644 --- a/pdd/operation_log.py +++ b/pdd/operation_log.py @@ -602,6 +602,11 @@ def save_fingerprint( logger.warning("Could not resolve paths for %s/%s: %s", basename, language, e) paths = {} + from .sync_core.finalize import finalize_legacy_paths + + if finalize_legacy_paths(paths): + return + path = get_fingerprint_path(basename, language, paths=paths) # Issue #522: Pass stored include deps for prompt hash calculation @@ -640,6 +645,10 @@ def save_run_report( Save a run report (test results) to the state file. `paths` (issue #1211) routes the file under the subproject meta dir. """ + from .sync_core.finalize import canonical_root_for_paths + + if canonical_root_for_paths(paths): + return path = get_run_report_path(basename, language, paths=paths) try: with open(path, 'w', encoding='utf-8') as f: diff --git a/pdd/pin_example_hack.py b/pdd/pin_example_hack.py index 15b4889cf..c28e7849d 100644 --- a/pdd/pin_example_hack.py +++ b/pdd/pin_example_hack.py @@ -222,6 +222,10 @@ def save_run_report(report: Dict[str, Any], basename: str, language: str, language: The programming language. atomic_state: Optional AtomicStateUpdate for atomic writes (Issue #159 fix). """ + from .sync_core.finalize import canonical_root_for_paths + + if canonical_root_for_paths(None): + return report_file = META_DIR / f"{_safe_basename(basename)}_{language.lower()}_run.json" if atomic_state: # Buffer for atomic write @@ -246,6 +250,10 @@ def _save_operation_fingerprint(basename: str, language: str, operation: str, model: The model used. atomic_state: Optional AtomicStateUpdate for atomic writes (Issue #159 fix). """ + from .sync_core.finalize import finalize_legacy_paths + + if finalize_legacy_paths(paths): + return from datetime import datetime, timezone from .sync_determine_operation import calculate_current_hashes, Fingerprint from . import __version__ @@ -1761,4 +1769,4 @@ def __init__(self, rc, out, err): PDD_DIR.mkdir(exist_ok=True) META_DIR.mkdir(exist_ok=True) result = sync_orchestration(basename="my_calculator", language="python", quiet=True) - print(json.dumps(result, indent=2)) \ No newline at end of file + print(json.dumps(result, indent=2)) diff --git a/pdd/preprocess.py b/pdd/preprocess.py index 8766130c3..7453a7a1d 100644 --- a/pdd/preprocess.py +++ b/pdd/preprocess.py @@ -6,13 +6,15 @@ import subprocess from typing import Any, List, Optional, Tuple, Union import traceback -from pathlib import Path +from pathlib import Path, PurePosixPath from rich.console import Console from rich.panel import Panel from rich.markup import escape from rich.traceback import install from .firecrawl_cache import get_firecrawl_cache from pdd.path_resolution import get_default_resolver +from pdd.sync_core.includes import include_paths +from pdd.sync_core.path_policy import PathPolicy install() console = Console() @@ -278,33 +280,7 @@ def compute_user_intent_paths(text: str) -> set: `compute_user_intent_paths(original_prompt)` with the result of `compute_user_intent_paths(_expand_vars(original_prompt, env_vars))`. """ - paths: set = set() - if not text: - return paths - # Match the full `` grammar that `process_include_tags` accepts — - # body form (with optional attrs) and self-closing form. Both forms can - # carry a `path="..."` attribute, which takes precedence over body content. - include_pattern = ( - r'\s[^>]*+)?>(?P.*?)' - r'|\s[^>]*?)/>' - ) - for m in re.finditer(include_pattern, text, flags=re.DOTALL): - attrs = _parse_attrs(m.group('attrs') or m.group('attrs_self') or "") - path_attr = attrs.get('path') - body = m.group('content') if m.group('content') is not None else "" - p = (path_attr or body).strip() - if p: - paths.add(p) - for m in re.finditer(r']*+)?>(.*?)', text, flags=re.DOTALL): - inner = m.group(1) - for raw in [s.strip() for part in inner.splitlines() for s in part.split(',')]: - if raw: - paths.add(raw) - for m in re.finditer(r"```<([^>]*+)>```", text): - p = m.group(1).strip() - if p: - paths.add(p) - return paths + return include_paths(text) def preprocess( @@ -456,9 +432,16 @@ def preprocess( def get_file_path(file_name: str) -> str: resolver = get_default_resolver() - resolved = resolver.resolve_include(file_name) - if not Path(file_name).is_absolute() and resolved == resolver.cwd / file_name: - return os.path.join("./", file_name) + project_root = resolver.resolve_project_root() + relpath = Path(file_name) + if relpath.is_absolute(): + raise ValueError(f"absolute include path is not allowed: {file_name}") + normalized = PurePosixPath(os.path.normpath(file_name).replace(os.sep, "/")) + resolved = PathPolicy(project_root).resolve( + normalized, allow_missing=True + ).canonical_path + if project_root.resolve() == Path.cwd().resolve(): + return os.path.join("./", normalized.as_posix()) return str(resolved) diff --git a/pdd/sync_core/__init__.py b/pdd/sync_core/__init__.py index 2c02990ad..3e3f12a96 100644 --- a/pdd/sync_core/__init__.py +++ b/pdd/sync_core/__init__.py @@ -6,6 +6,8 @@ CheckerIdentity, GlobalCertificateOptions, LifecycleResult, + NightlyObservation, + RemoteCertificateSigner, RepositoryTarget, build_global_certificate, count_vendored_sync_semantics, @@ -85,8 +87,10 @@ AttestationBinding, AttestationEnvelope, AttestationError, + AttestationIssuer, AttestationRequest, AttestationSigner, + RemoteAttestationSigner, AttestationTrustPolicy, FileReplayStore, InMemoryReplayStore, @@ -128,9 +132,11 @@ "AttestationBinding", "AttestationEnvelope", "AttestationError", + "AttestationIssuer", "AttestationIssue", "AttestationRequest", "AttestationSigner", + "RemoteAttestationSigner", "AttestationTrustPolicy", "BaselineStatus", "CandidateId", @@ -164,6 +170,8 @@ "LegacyFingerprintRecord", "LoadedTrustPolicy", "LifecycleResult", + "NightlyObservation", + "RemoteCertificateSigner", "ManifestError", "ManifestUnit", "PathPolicy", diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index 37fdca7cc..d0eabaa54 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -12,13 +12,56 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path, PurePosixPath -from typing import Any +from typing import Any, Protocol from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey from .reporting import CanonicalReportOptions, build_canonical_report -from .trust import AttestationSigner + + +class CertificateSigner(Protocol): + """Minimal signer surface implemented by protected remote authorities.""" + + issuer: str + + def public_key_bytes(self) -> bytes: + """Return the pinned Ed25519 public key.""" + + def sign_bytes(self, payload: bytes) -> str: + """Return a base64 Ed25519 signature from the remote authority.""" + + +class RemoteCertificateSigner: + """Invoke a protected KMS/keyless signer without loading private key bytes.""" + + def __init__(self, issuer: str, public_key: bytes, command: tuple[str, ...]) -> None: + if not issuer or len(public_key) != 32 or not command: + raise ValueError("remote certificate signer configuration is invalid") + self.issuer = issuer + self._public_key = public_key + self._command = command + + def public_key_bytes(self) -> bytes: + """Return the protected expected public key.""" + return self._public_key + + def sign_bytes(self, payload: bytes) -> str: + """Sign canonical bytes remotely and verify the response locally.""" + result = subprocess.run( + self._command, + input=payload, + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise ValueError("remote certificate signer rejected the request") + try: + signature = base64.b64decode(result.stdout.strip(), validate=True) + Ed25519PublicKey.from_public_bytes(self._public_key).verify(signature, payload) + except (ValueError, InvalidSignature) as exc: + raise ValueError("remote certificate signer returned an invalid signature") from exc + return base64.b64encode(signature).decode("ascii") @dataclass(frozen=True) @@ -70,6 +113,29 @@ def payload(self) -> dict[str, str]: } +@dataclass(frozen=True) +class NightlyObservation: + """Externally observed temporal checks bound into a signed nightly row.""" + + complete_scan: bool + ledgers_deleted_before_scan: bool + normal_scan_writes: int + injected_canary_detected: bool + injected_canary_outcome: str + post_canary_rerun_writes: int + + def payload(self) -> dict[str, bool | int | str]: + """Return the canonical signed observation payload.""" + return { + "complete_scan": self.complete_scan, + "ledgers_deleted_before_scan": self.ledgers_deleted_before_scan, + "normal_scan_writes": self.normal_scan_writes, + "injected_canary_detected": self.injected_canary_detected, + "injected_canary_outcome": self.injected_canary_outcome, + "post_canary_rerun_writes": self.post_canary_rerun_writes, + } + + @dataclass(frozen=True) class GlobalCertificateOptions: """Trust and external evidence inputs for global certification.""" @@ -79,6 +145,7 @@ class GlobalCertificateOptions: nightly_ledger: Path required_nightly_streak: int = 7 checker_identity: CheckerIdentity | None = None + nightly_observation: NightlyObservation | None = None def _canonical_bytes(payload: dict[str, Any]) -> bytes: @@ -330,6 +397,7 @@ def _complete_nightly( return ( required_counts <= counts.keys() and required_lifecycle <= lifecycle.keys() + and _nightly_observation_complete(row.get("nightly_observation")) and row.get("checker") == checker_identity.payload() and _nightly_lineage(row, targets) and _verify_certificate_integrity( @@ -338,6 +406,22 @@ def _complete_nightly( ) +def _nightly_observation_complete(payload: Any) -> bool: + """Validate the signed observations required for one temporal row.""" + if not isinstance(payload, dict): + return False + return ( + payload.get("complete_scan") is True + and payload.get("ledgers_deleted_before_scan") is True + and payload.get("normal_scan_writes") == 0 + and not isinstance(payload.get("normal_scan_writes"), bool) + and payload.get("injected_canary_detected") is True + and payload.get("injected_canary_outcome") in {"REPAIRED", "BLOCKED"} + and payload.get("post_canary_rerun_writes") == 0 + and not isinstance(payload.get("post_canary_rerun_writes"), bool) + ) + + def _nightly_streak( path: Path, public_key: bytes, @@ -435,6 +519,7 @@ def _scan_predicate( and extra["pdd_cloud_vendored_sync_semantics"] == 0 and lifecycle.post_repair_second_run_writes == 0 and lifecycle.post_merge_tree_changes == 0 + and extra["nightly_observation_complete"] == 1 ) @@ -568,6 +653,7 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] "pdd_cloud_vendored_sync_semantics", "nightly_streak", "required_nightly_streak", + "nightly_observation_complete", } if any( not isinstance(counts_payload.get(name), int) @@ -577,6 +663,10 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] ): return False, False extra = {name: counts_payload[name] for name in extra_names} + if extra["nightly_observation_complete"] != int( + _nightly_observation_complete(body.get("nightly_observation")) + ): + return False, False expected_profile_coverage = ( 100 if aggregate["managed_units"] > 0 @@ -605,7 +695,7 @@ def _build_global_certificate_from_targets( targets: tuple[RepositoryTarget, ...], options: GlobalCertificateOptions, *, - signer: AttestationSigner, + signer: CertificateSigner, ) -> dict[str, Any]: """Build and sign the complete cross-repository machine predicate.""" if not targets: @@ -641,12 +731,21 @@ def _build_global_certificate_from_targets( options.checker_identity, ), "required_nightly_streak": options.required_nightly_streak, + "nightly_observation_complete": int( + options.nightly_observation is not None + and _nightly_observation_complete(options.nightly_observation.payload()) + ), } lifecycle = options.lifecycle_result body: dict[str, Any] = { "schema_version": 2, "checked_at": datetime.now(timezone.utc).isoformat(), "checker": options.checker_identity.payload(), + "nightly_observation": ( + options.nightly_observation.payload() + if options.nightly_observation is not None + else None + ), "repositories": reports, "counts": { **counts, @@ -694,7 +793,7 @@ def build_global_certificate( targets: tuple[RepositoryTarget, ...], options: GlobalCertificateOptions, *, - signer: AttestationSigner, + signer: CertificateSigner, ) -> dict[str, Any]: """Build from independent exact-SHA clones and revalidate before signing.""" if not targets: @@ -791,17 +890,25 @@ def verify_global_certificate( ) -def signer_from_environment() -> AttestationSigner: - """Load certificate signing identity from protected runner environment.""" - encoded = os.environ.get("PDD_CERTIFICATE_SIGNING_KEY") - issuer = os.environ.get("PDD_CERTIFICATE_ISSUER", "global-sync-checker") - if not encoded: - raise ValueError("PDD_CERTIFICATE_SIGNING_KEY is required") +def signer_from_environment() -> CertificateSigner: + """Load a remote signing authority without accepting local private keys.""" + if os.environ.get("PDD_CERTIFICATE_SIGNING_KEY"): + raise ValueError("local certificate signing keys are forbidden") + encoded = os.environ.get("PDD_CERTIFICATE_PUBLIC_KEY") + issuer = os.environ.get("PDD_CERTIFICATE_ISSUER", "") + command_raw = os.environ.get("PDD_CERTIFICATE_SIGNER_COMMAND", "") + if not encoded or not issuer or not command_raw: + raise ValueError("protected remote certificate signer is required") try: - key = base64.b64decode(encoded, validate=True) - except ValueError as exc: - raise ValueError("PDD_CERTIFICATE_SIGNING_KEY is malformed") from exc - return AttestationSigner(issuer, key) + public_key = base64.b64decode(encoded, validate=True) + command_payload = json.loads(command_raw) + except (ValueError, json.JSONDecodeError) as exc: + raise ValueError("remote certificate signer configuration is malformed") from exc + if not isinstance(command_payload, list) or not all( + isinstance(item, str) and item for item in command_payload + ): + raise ValueError("remote certificate signer command is malformed") + return RemoteCertificateSigner(issuer, public_key, tuple(command_payload)) def checker_identity_from_environment( diff --git a/pdd/sync_core/decommission.py b/pdd/sync_core/decommission.py new file mode 100644 index 000000000..6dd98d020 --- /dev/null +++ b/pdd/sync_core/decommission.py @@ -0,0 +1,102 @@ +"""Protected expected-unit and decommission authorization parsing.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +from .git_io import read_git_blob +from .types import UnitId + + +@dataclass(frozen=True) +class DecommissionTombstone: + """Protected proof that a synchronized unit was deliberately removed.""" + + prompt_path: PurePosixPath + artifact_paths: tuple[PurePosixPath, ...] + rationale: str + owner: str + baseline_status: str + + +def load_tombstones( + root: Path, ref: str +) -> dict[PurePosixPath, DecommissionTombstone]: + """Load strict decommission authorizations from one immutable Git tree.""" + raw = read_git_blob(root, ref, PurePosixPath(".pdd/sync-tombstones.json")) + if raw is None: + return {} + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ValueError("protected sync tombstones are malformed") from exc + if not isinstance(payload, list): + raise ValueError("protected sync tombstones must be a list") + tombstones: dict[PurePosixPath, DecommissionTombstone] = {} + for item in payload: + if not isinstance(item, dict): + raise ValueError("each sync tombstone must be an object") + try: + prompt_path = PurePosixPath(item["prompt_path"]) + artifacts = tuple( + sorted(PurePosixPath(value) for value in item["artifact_paths"]) + ) + tombstone = DecommissionTombstone( + prompt_path, + artifacts, + item["rationale"], + item["owner"], + item["baseline_status"], + ) + except (KeyError, TypeError) as exc: + raise ValueError("sync tombstone is missing required fields") from exc + if ( + prompt_path.is_absolute() + or ".." in prompt_path.parts + or any(path.is_absolute() or ".." in path.parts for path in artifacts) + or not tombstone.rationale + or not tombstone.owner + ): + raise ValueError("sync tombstone contains invalid protected fields") + if prompt_path in tombstones: + raise ValueError("duplicate sync tombstone prompt identity") + tombstones[prompt_path] = tombstone + return tombstones + + +def load_expected_registry( + root: Path, ref: str, repository_id: str +) -> set[UnitId] | None: + """Load the protected active-unit denominator when one has been established.""" + raw = read_git_blob(root, ref, PurePosixPath(".pdd/expected-managed.json")) + if raw is None: + return None + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ValueError("protected expected-managed registry is malformed") from exc + rows = payload.get("units") if isinstance(payload, dict) else None + if ( + not isinstance(payload, dict) + or payload.get("schema_version") != 1 + or not isinstance(rows, list) + ): + raise ValueError("protected expected-managed registry schema is invalid") + units: set[UnitId] = set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"prompt_path", "language_id"}: + raise ValueError("expected-managed unit entry is malformed") + try: + unit = UnitId( + repository_id, + PurePosixPath(row["prompt_path"]), + row["language_id"], + ) + except (TypeError, ValueError) as exc: + raise ValueError("expected-managed unit identity is invalid") from exc + if unit in units: + raise ValueError("expected-managed registry contains a duplicate unit") + units.add(unit) + return units diff --git a/pdd/sync_core/finalize.py b/pdd/sync_core/finalize.py index 49597f8ed..327cf0c54 100644 --- a/pdd/sync_core/finalize.py +++ b/pdd/sync_core/finalize.py @@ -4,6 +4,7 @@ import base64 import hashlib +import json import os import subprocess import uuid @@ -18,6 +19,7 @@ load_trust_policy, ) from .fingerprint_store import FingerprintStore, encode_fingerprint +from .git_io import resolve_git_commit from .manifest import build_unit_manifest from .runner import ( TRUSTED_RUNNER_VERSION, @@ -35,7 +37,7 @@ TransactionPhase, TransactionResult, ) -from .trust import AttestationBinding, AttestationSigner +from .trust import AttestationBinding, AttestationIssuer, RemoteAttestationSigner from .types import ( EvidenceOutcome, FingerprintProvenance, @@ -64,8 +66,9 @@ def canonical_root_for_paths(paths: dict[str, Path] | None) -> Path | None: from ..continuous_sync import canonical_sync_enabled, repository_root start = Path(paths.get("prompt", Path.cwd())) if paths else Path.cwd() - root = repository_root(start) - return root if canonical_sync_enabled(root) else None + if not canonical_sync_enabled(start): + return None + return repository_root(start) def finalize_legacy_paths(paths: dict[str, Path] | None) -> bool: @@ -95,30 +98,25 @@ def finalize_legacy_paths(paths: dict[str, Path] | None) -> bool: return True -def _git_sha(root: Path, ref: str) -> str: - result = subprocess.run( - ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], - cwd=root, - capture_output=True, - text=True, - check=False, - ) - if result.returncode != 0 or not result.stdout.strip(): - raise ValueError(f"cannot resolve Git commit: {ref}") - return result.stdout.strip() - - -def attestation_signer_from_environment() -> AttestationSigner: - """Load the protected semantic-evidence signing identity.""" - encoded = os.environ.get("PDD_ATTESTATION_SIGNING_KEY") - issuer = os.environ.get("PDD_ATTESTATION_ISSUER", "trusted-sync-runner") - if not encoded: - raise ValueError("PDD_ATTESTATION_SIGNING_KEY is required") +def attestation_signer_from_environment() -> AttestationIssuer: + """Load a remote evidence authority without accepting local private keys.""" + if os.environ.get("PDD_ATTESTATION_SIGNING_KEY"): + raise ValueError("local attestation signing keys are forbidden") + encoded = os.environ.get("PDD_ATTESTATION_PUBLIC_KEY") + issuer = os.environ.get("PDD_ATTESTATION_ISSUER", "") + command_raw = os.environ.get("PDD_ATTESTATION_SIGNER_COMMAND", "") + if not encoded or not issuer or not command_raw: + raise ValueError("protected remote attestation signer is required") try: - key = base64.b64decode(encoded, validate=True) - except ValueError as exc: - raise ValueError("PDD_ATTESTATION_SIGNING_KEY is malformed") from exc - return AttestationSigner(issuer, key) + public_key = base64.b64decode(encoded, validate=True) + command_payload = json.loads(command_raw) + except (ValueError, json.JSONDecodeError) as exc: + raise ValueError("remote attestation signer configuration is malformed") from exc + if not isinstance(command_payload, list) or not all( + isinstance(item, str) and item for item in command_payload + ): + raise ValueError("remote attestation signer command is malformed") + return RemoteAttestationSigner(issuer, public_key, tuple(command_payload)) def _artifact_writes(root: Path, snapshot) -> tuple[PlannedWrite, ...]: @@ -222,16 +220,16 @@ def finalize_unit( *, base_ref: str, head_ref: str, - signer: AttestationSigner, + signer: AttestationIssuer, config: RunnerConfig = RunnerConfig(), replay_ledger_path: Path | None = None, ) -> FinalizeResult: # pylint: disable=too-many-arguments,too-many-locals """Validate one complete unit and atomically finalize all trusted state.""" repository_root = Path(root).resolve() - base_sha = _git_sha(repository_root, base_ref) - head_sha = _git_sha(repository_root, head_ref) - if _git_sha(repository_root, "HEAD") != head_sha: + base_sha = resolve_git_commit(repository_root, base_ref) + head_sha = resolve_git_commit(repository_root, head_ref) + if resolve_git_commit(repository_root, "HEAD") != head_sha: raise ValueError("canonical finalization requires HEAD at the checked SHA") manifest = build_unit_manifest( repository_root, base_ref=base_sha, head_ref=head_sha diff --git a/pdd/sync_core/git_io.py b/pdd/sync_core/git_io.py index 6adde8424..2f16eaeea 100644 --- a/pdd/sync_core/git_io.py +++ b/pdd/sync_core/git_io.py @@ -13,3 +13,17 @@ def read_git_blob(root: Path, ref: str, path: PurePosixPath) -> bytes | None: check=False, ) return result.stdout if result.returncode == 0 else None + + +def resolve_git_commit(root: Path, ref: str) -> str: + """Resolve one exact commit or fail closed.""" + result = subprocess.run( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise ValueError(f"cannot resolve Git commit: {ref}") + return result.stdout.strip() diff --git a/pdd/sync_core/isolation.py b/pdd/sync_core/isolation.py new file mode 100644 index 000000000..789f11fcb --- /dev/null +++ b/pdd/sync_core/isolation.py @@ -0,0 +1,10 @@ +"""Shared credential-removal policy for untrusted child processes.""" + +SECRET_ENV_MARKERS = ( + "API_KEY", + "CREDENTIAL", + "PASSWORD", + "SECRET", + "SIGNING_KEY", + "TOKEN", +) diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 6ca564049..37e2c129c 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -11,25 +11,16 @@ from typing import Any from .certificate import LifecycleResult +from .isolation import SECRET_ENV_MARKERS from .scenario_contract import REQUIRED_SCENARIOS -_SECRET_ENV_MARKERS = ( - "API_KEY", - "CREDENTIAL", - "PASSWORD", - "SECRET", - "SIGNING_KEY", - "TOKEN", -) - - def _isolated_lifecycle_environment(home: Path) -> dict[str, str]: """Build a credential-free environment with no source import overrides.""" environment = { key: value for key, value in os.environ.items() - if not any(marker in key.upper() for marker in _SECRET_ENV_MARKERS) + if not any(marker in key.upper() for marker in SECRET_ENV_MARKERS) and key not in {"PYTHONPATH", "PYTHONHOME", "PDD_PATH"} } environment["HOME"] = str(home) @@ -85,9 +76,44 @@ def _normalized_results(payload: Any) -> dict[str, dict[str, Any]]: return results +def _install_candidate_wheel(temporary: Path, home: Path, wheel: Path) -> Path | None: + """Install the exact candidate wheel in a separate isolated environment.""" + environment = temporary / "candidate-venv" + isolated = _isolated_lifecycle_environment(home) + created = subprocess.run( + [sys.executable, "-m", "venv", "--system-site-packages", str(environment)], + capture_output=True, + text=True, + check=False, + env=isolated, + ) + candidate_python = environment / ( + "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + if created.returncode != 0: + return None + installed = subprocess.run( + [ + str(candidate_python), + "-m", + "pip", + "install", + "--no-deps", + "--force-reinstall", + str(wheel.resolve()), + ], + capture_output=True, + text=True, + check=False, + env=isolated, + ) + return candidate_python if installed.returncode == 0 else None + + def run_lifecycle_matrix( root: Path, *, + candidate_wheel: Path | None = None, cloud_root: Path | None = None, cloud_base_ref: str | None = None, cloud_head_ref: str | None = None, @@ -96,13 +122,23 @@ def run_lifecycle_matrix( # pylint: disable=too-many-arguments """Run only the scenario harness installed with the released checker.""" del root # Candidate repository tests are never lifecycle evidence. - if cloud_root is None or cloud_base_ref is None or cloud_head_ref is None: + if ( + candidate_wheel is None + or not Path(candidate_wheel).is_file() + or cloud_root is None + or cloud_base_ref is None + or cloud_head_ref is None + ): return _failed_result() with tempfile.TemporaryDirectory(prefix="pdd-released-lifecycle-") as directory: temporary = Path(directory) output = temporary / "result.json" - home = temporary / "home" - home.mkdir(mode=0o700) + (temporary / "home").mkdir(mode=0o700) + candidate_python = _install_candidate_wheel( + temporary, temporary / "home", Path(candidate_wheel) + ) + if candidate_python is None: + return _failed_result() command = [ sys.executable, "-I", @@ -116,6 +152,8 @@ def run_lifecycle_matrix( cloud_base_ref, "--cloud-head-ref", cloud_head_ref, + "--candidate-python", + str(candidate_python), ] try: completed = subprocess.run( @@ -124,7 +162,7 @@ def run_lifecycle_matrix( text=True, check=False, timeout=timeout_seconds, - env=_isolated_lifecycle_environment(home), + env=_isolated_lifecycle_environment(temporary / "home"), ) except subprocess.TimeoutExpired: return _failed_result(timeout=True) diff --git a/pdd/sync_core/manifest.py b/pdd/sync_core/manifest.py index 1cfee29ab..5730f4c9a 100644 --- a/pdd/sync_core/manifest.py +++ b/pdd/sync_core/manifest.py @@ -13,6 +13,11 @@ import yaml +from .decommission import ( + DecommissionTombstone, + load_expected_registry, + load_tombstones, +) from .identity import REPOSITORY_ID_RELPATH, canonical_repository_id from .git_io import read_git_blob from .language import LanguageRegistry, LanguageRegistryError @@ -68,17 +73,6 @@ class ManifestRefs: head: str -@dataclass(frozen=True) -class DecommissionTombstone: - """Protected proof that a synchronized unit was deliberately removed.""" - - prompt_path: PurePosixPath - artifact_paths: tuple[PurePosixPath, ...] - rationale: str - owner: str - baseline_status: str - - @dataclass(frozen=True, order=True) class OwnershipRule: """Protected-base classification for an otherwise unmatched tracked path.""" @@ -193,6 +187,50 @@ def digest(self) -> str: encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() return hashlib.sha256(encoded).hexdigest() + def unit_digest(self, unit: ManifestUnit) -> str: + """Bind one unit to its own manifest slice and relevant policy.""" + if unit not in self.units: + raise ValueError("unit is not part of this manifest") + owned_paths = {unit.unit_id.prompt_relpath, *unit.artifact_paths} + candidates = [ + item + for item in self.candidates + if not _is_dynamic_canonical_state(item.candidate_id.artifact_relpath) + and ( + item.unit_id == unit.unit_id + or item.candidate_id.artifact_relpath in owned_paths + ) + ] + payload = { + "repository_id": self.repository_id, + "language_registry_digest": self.language_registry_digest, + "unit": { + "id": _unit_payload(unit.unit_id), + "present_in_base": unit.present_in_base, + "present_in_head": unit.present_in_head, + "artifact_paths": [path.as_posix() for path in unit.artifact_paths], + "tombstoned": unit.tombstoned, + }, + "candidates": [ + { + "path": item.candidate_id.artifact_relpath.as_posix(), + "role": item.candidate_id.role, + "inventory": item.inventory.value, + "in_base": item.in_base, + "in_head": item.in_head, + "provenance": item.ownership_provenance, + "base_object_id": item.base_object_id, + "base_git_mode": item.base_git_mode, + "head_object_id": item.head_object_id, + "head_git_mode": item.head_git_mode, + "unit": _unit_payload(item.unit_id), + } + for item in candidates + ], + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + def _unit_payload(unit_id: UnitId | None) -> dict[str, str] | None: if unit_id is None: @@ -530,50 +568,6 @@ def _map_architecture_modules( return outputs, invalid -def _tombstones( - root: Path, - ref: str, - entries: dict[PurePosixPath, GitTreeEntry], -) -> dict[PurePosixPath, DecommissionTombstone]: - path = PurePosixPath(".pdd/sync-tombstones.json") - if path not in entries: - return {} - try: - payload = json.loads(_blob(root, ref, path)) - except (json.JSONDecodeError, UnicodeDecodeError) as exc: - raise ManifestError("protected sync tombstones are malformed") from exc - if not isinstance(payload, list): - raise ManifestError("protected sync tombstones must be a list") - tombstones: dict[PurePosixPath, DecommissionTombstone] = {} - for item in payload: - if not isinstance(item, dict): - raise ManifestError("each sync tombstone must be an object") - try: - prompt_path = PurePosixPath(item["prompt_path"]) - artifacts = tuple(sorted(PurePosixPath(value) for value in item["artifact_paths"])) - tombstone = DecommissionTombstone( - prompt_path, - artifacts, - item["rationale"], - item["owner"], - item["baseline_status"], - ) - except (KeyError, TypeError) as exc: - raise ManifestError("sync tombstone is missing required fields") from exc - if ( - prompt_path.is_absolute() - or ".." in prompt_path.parts - or any(path.is_absolute() or ".." in path.parts for path in artifacts) - or not tombstone.rationale - or not tombstone.owner - ): - raise ManifestError("sync tombstone contains invalid protected fields") - if prompt_path in tombstones: - raise ManifestError("duplicate sync tombstone prompt identity") - tombstones[prompt_path] = tombstone - return tombstones - - def _manifest_unit( prompt_path: PurePosixPath, sources: _UnitSources, @@ -601,8 +595,11 @@ def _manifest_unit( f"{head_ref}:{prompt_path.as_posix()}: removed managed unit lacks " "a complete IN_SYNC tombstone" ) - elif tombstone and not removed: - reason = f"{head_ref}:{prompt_path.as_posix()}: tombstone targets a present unit" + elif tombstone and ( + set(tombstone.artifact_paths) != base_artifacts | {prompt_path} + or tombstone.baseline_status != "IN_SYNC" + ): + reason = f"{head_ref}:{prompt_path.as_posix()}: decommission authorization is incomplete" unit = ManifestUnit( unit_id, prompt_path in sources.base_units, @@ -617,6 +614,7 @@ def _manifest_units( sources: _UnitSources, tombstones: dict[PurePosixPath, DecommissionTombstone], head_ref: str, + registry_paths: set[PurePosixPath], ) -> tuple[list[ManifestUnit], list[str]]: """Assemble units and validate protected decommission transitions.""" units: list[ManifestUnit] = [] @@ -629,7 +627,7 @@ def _manifest_units( units.append(unit) if reason: invalid.append(reason) - unknown = set(tombstones) - set(sources.base_units) + unknown = set(tombstones) - set(sources.base_units) - registry_paths invalid.extend( f"{head_ref}:{path.as_posix()}: tombstone has no base unit" for path in sorted(unknown) @@ -712,6 +710,8 @@ def _is_protected_control(path: PurePosixPath) -> bool: or path in { PurePosixPath(".pdd/repository-id"), + PurePosixPath(".pdd/expected-managed.json"), + PurePosixPath(".pdd/sync-policy.json"), PurePosixPath(".pdd/verification-profiles.json"), PurePosixPath(".pdd/attestation-trust.json"), PurePosixPath(".pdd/sync-ownership.json"), @@ -844,21 +844,65 @@ def _tree_manifest( ) +def _protected_expected_units( + units: list[ManifestUnit], + tombstones: dict[PurePosixPath, DecommissionTombstone], + registry: set[UnitId] | None, +) -> tuple[set[UnitId], list[str]]: + """Apply the protected denominator and authorized removal transitions.""" + discovered = {unit.unit_id for unit in units} + if registry is None: + return discovered, [] + missing_registry = { + unit.unit_id + for unit in units + if unit.present_in_base and unit.unit_id not in registry + } + authorized_removed = { + item + for item in registry + if item.prompt_relpath in tombstones + and not any(unit.unit_id == item and unit.present_in_head for unit in units) + } + head_additions = { + unit.unit_id + for unit in units + if unit.present_in_head and not unit.present_in_base + } + expected = (registry - authorized_removed) | head_additions + unresolved = registry - discovered - authorized_removed + invalid = [ + *( + f"{item.prompt_relpath}: protected expected-managed registry omits base unit" + for item in sorted(missing_registry) + ), + *( + f"{item.prompt_relpath}: expected managed unit is absent without authorization" + for item in sorted(unresolved) + ), + ] + return expected, invalid + + def _assemble_manifest( repository_id: str, language_registry_digest: str, base: _TreeManifest, head: _TreeManifest, tombstones: dict[PurePosixPath, DecommissionTombstone], + expected_registry: set[UnitId] | None, ownership_rules: tuple[OwnershipRule, ...], ) -> UnitManifest: # pylint: disable=too-many-arguments,too-many-positional-arguments """Combine parsed trees into the final candidate and unit partition.""" invalid = list(base.invalid_reasons + head.invalid_reasons) - sources = _UnitSources(base.units, head.units, base.outputs, head.outputs) - units, tombstone_invalid = _manifest_units(sources, tombstones, head.ref) + units, tombstone_invalid = _manifest_units( + _UnitSources(base.units, head.units, base.outputs, head.outputs), + tombstones, + head.ref, + {item.prompt_relpath for item in (expected_registry or set())}, + ) invalid.extend(tombstone_invalid) - all_paths = set(base.entries) | set(head.entries) candidates, accounted, ownership_invalid = _candidate_records( _CandidateSources( repository_id, @@ -870,15 +914,19 @@ def _assemble_manifest( ) ) invalid.extend(ownership_invalid) + expected, expected_invalid = _protected_expected_units( + units, tombstones, expected_registry + ) + invalid.extend(expected_invalid) return UnitManifest( repository_id, language_registry_digest, ManifestRefs(base.ref, head.ref), tuple(candidates), tuple(sorted(units)), - tuple(sorted(unit.unit_id for unit in units)), + tuple(sorted(expected)), tuple(sorted(invalid)), - tuple(sorted(all_paths - accounted)), + tuple(sorted((set(base.entries) | set(head.entries)) - accounted)), ) @@ -918,12 +966,21 @@ def build_unit_manifest( ownership, set(base.entries), ) - tombstones = _tombstones(repository_root, head_ref, head.entries) + try: + tombstones = load_tombstones(repository_root, base_ref) + load_tombstones(repository_root, head_ref) + expected_registry = load_expected_registry( + repository_root, base_ref, repository_id + ) + load_expected_registry(repository_root, head_ref, repository_id) + except ValueError as exc: + raise ManifestError(str(exc)) from exc return _assemble_manifest( repository_id, language_registry.digest(), base, head, tombstones, + expected_registry, ownership, ) diff --git a/pdd/sync_core/reporting.py b/pdd/sync_core/reporting.py index 2a6485893..db5988a8f 100644 --- a/pdd/sync_core/reporting.py +++ b/pdd/sync_core/reporting.py @@ -16,6 +16,7 @@ load_trust_policy, ) from .fingerprint_store import CorruptFingerprintError, FingerprintStore +from .git_io import resolve_git_commit from .manifest import ManifestUnit, UnitManifest, build_unit_manifest from .snapshot import SnapshotError, build_unit_snapshot from .runner import TRUSTED_RUNNER_VERSION, runner_identity_digest @@ -115,19 +116,6 @@ class CanonicalReportOptions: now: datetime | None = None -def _git_sha(root: Path, ref: str) -> str: - result = subprocess.run( - ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], - cwd=root, - capture_output=True, - text=True, - check=False, - ) - if result.returncode != 0 or not result.stdout.strip(): - raise ValueError(f"cannot resolve Git commit: {ref}") - return result.stdout.strip() - - def _error_verdict(unit: ManifestUnit, baseline: BaselineStatus, reason: str) -> SyncVerdict: return SyncVerdict( unit.unit_id, @@ -268,8 +256,8 @@ def _report_context( options: CanonicalReportOptions, ) -> tuple[ReportContext, list[str], tuple[str, ...]]: """Resolve protected inputs, trust policy, store, and recovery state.""" - base_sha = _git_sha(root, options.base_ref) - head_sha = _git_sha(root, options.head_ref) + base_sha = resolve_git_commit(root, options.base_ref) + head_sha = resolve_git_commit(root, options.head_ref) manifest = build_unit_manifest(root, base_ref=base_sha, head_ref=head_sha) profiles = load_verification_profiles(root, manifest) now = options.now or datetime.now(timezone.utc) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index e0e222093..765a751a0 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -18,9 +18,11 @@ from .trust import ( AttestationBinding, AttestationEnvelope, + AttestationIssuer, AttestationRequest, - AttestationSigner, ) +from .isolation import SECRET_ENV_MARKERS +from .git_io import read_git_blob from .types import ( EvidenceOutcome, ObligationEvidence, @@ -70,22 +72,12 @@ class RunBinding: class AttestationIssue: """Trusted signer and unique issuance fields for one profile run.""" - signer: AttestationSigner + signer: AttestationIssuer attestation_id: str nonce: str issued_at: datetime -def _git_blob(root: Path, ref: str, path: PurePosixPath) -> bytes | None: - result = subprocess.run( - ["git", "show", f"{ref}:{path.as_posix()}"], - cwd=root, - capture_output=True, - check=False, - ) - return result.stdout if result.returncode == 0 else None - - def _local_module_paths( root: Path, ref: str, source_path: PurePosixPath, source: bytes ) -> set[PurePosixPath]: @@ -125,7 +117,9 @@ def _local_module_paths( module_path.with_suffix(".py"), module_path / "__init__.py", ) - resolved.update(path for path in candidates if _git_blob(root, ref, path) is not None) + resolved.update( + path for path in candidates if read_git_blob(root, ref, path) is not None + ) return resolved @@ -133,35 +127,79 @@ def _pytest_support_closure( root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] ) -> tuple[tuple[PurePosixPath, bytes], ...]: """Return config, conftest, and transitive local-import blobs for pytest.""" - pending = list(test_paths) - paths = {path for path in PYTEST_CONFIG_PATHS if _git_blob(root, ref, path) is not None} + config_paths = _pytest_config_paths(root, ref, test_paths) + return _transitive_support_blobs( + root, + ref, + pending=tuple(test_paths) + tuple(config_paths), + included=config_paths, + ) + + +def _pytest_config_paths( + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] +) -> set[PurePosixPath]: + """Return protected pytest config and governing conftest paths.""" + paths = { + path for path in PYTEST_CONFIG_PATHS + if read_git_blob(root, ref, path) is not None + } for test_path in test_paths: parent = test_path.parent while parent != PurePosixPath("."): candidate = parent / "conftest.py" - if _git_blob(root, ref, candidate) is not None: + if read_git_blob(root, ref, candidate) is not None: paths.add(candidate) parent = parent.parent root_conftest = PurePosixPath("conftest.py") - if _git_blob(root, ref, root_conftest) is not None: + if read_git_blob(root, ref, root_conftest) is not None: paths.add(root_conftest) - pending.extend(paths) + return paths + + +def _transitive_support_blobs( + root: Path, + ref: str, + *, + pending: tuple[PurePosixPath, ...], + included: set[PurePosixPath], +) -> tuple[tuple[PurePosixPath, bytes], ...]: + """Resolve local Python imports from protected runner support paths.""" + paths = set(included) + remaining = list(pending) visited: set[PurePosixPath] = set() - while pending: - path = pending.pop() + while remaining: + path = remaining.pop() if path in visited: continue visited.add(path) - source = _git_blob(root, ref, path) + source = read_git_blob(root, ref, path) if source is None or path.suffix != ".py": continue discovered = _local_module_paths(root, ref, path, source) - visited paths.update(discovered) - pending.extend(discovered) - blobs = ((path, _git_blob(root, ref, path)) for path in sorted(paths)) + remaining.extend(discovered) + blobs = ((path, read_git_blob(root, ref, path)) for path in sorted(paths)) return tuple((path, blob) for path, blob in blobs if blob is not None) +def pytest_validator_config_digest( + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] +) -> str: + """Hash the actual pytest config/conftest closure declared by a profile.""" + config_paths = _pytest_config_paths(root, ref, test_paths) + closure = _transitive_support_blobs( + root, + ref, + pending=tuple(config_paths), + included=config_paths, + ) + digest = hashlib.sha256() + for path, content in closure: + digest.update(path.as_posix().encode() + b"\0" + content + b"\0") + return digest.hexdigest() + + def _support_digest( root: Path, ref: str, profile: VerificationProfile ) -> tuple[str, tuple[PurePosixPath, ...]]: @@ -182,7 +220,7 @@ def _config_loads_plugin(root: Path, ref: str) -> bool: """Reject repository-configured plugins until profiles bind plugin identities.""" pattern = re.compile(r"(?:^|[\s\"'])-p(?:[=\s]+)[A-Za-z0-9_.-]+") for path in PYTEST_CONFIG_PATHS: - content = _git_blob(root, ref, path) + content = read_git_blob(root, ref, path) if content is None: continue try: @@ -315,16 +353,7 @@ def _pytest_environment() -> dict[str, str]: return { key: value for key, value in os.environ.items() - if not any( - marker in key.upper() - for marker in ( - "CREDENTIAL", - "PASSWORD", - "SECRET", - "SIGNING_KEY", - "TOKEN", - ) - ) + if not any(marker in key.upper() for marker in SECRET_ENV_MARKERS) and key not in {"PYTEST_ADDOPTS", "PYTHONPATH"} } | {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PYTHONNOUSERSITE": "1"} @@ -559,6 +588,17 @@ def _obligation_preflight( obligation.validator_config_digest, "test obligation declares no artifact paths", ) + expected_config_digests = { + pytest_validator_config_digest(root, ref, obligation.artifact_paths) + for ref in (base_sha, head_sha) + } + if expected_config_digests != {obligation.validator_config_digest}: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "declared validator config digest does not match protected closure", + ) if _config_loads_plugin(root, base_sha) or _config_loads_plugin(root, head_sha): return RunnerExecution( obligation.obligation_id, diff --git a/pdd/sync_core/scenario_contract.py b/pdd/sync_core/scenario_contract.py index b52e692b7..faa9ac259 100644 --- a/pdd/sync_core/scenario_contract.py +++ b/pdd/sync_core/scenario_contract.py @@ -11,6 +11,8 @@ "transactional-canonical-report", "merge-group-base-movement-and-stale-repair", "built-wheel-clean-environment", + "candidate-wheel-public-report", + "candidate-wheel-transaction-recovery", "pdd-cloud-real-consumer-canary", "released-checker-owned-scenario-harness", } diff --git a/pdd/sync_core/scenario_harness.py b/pdd/sync_core/scenario_harness.py index 0f6cb5c85..6ce3d7b66 100644 --- a/pdd/sync_core/scenario_harness.py +++ b/pdd/sync_core/scenario_harness.py @@ -9,7 +9,7 @@ import tempfile import threading from concurrent.futures import ThreadPoolExecutor -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from datetime import datetime, timedelta, timezone from pathlib import Path, PurePosixPath from typing import Callable @@ -18,7 +18,6 @@ from .classifier import classify from .identity import initialize_repository_identity from .manifest import build_unit_manifest -from .reporting import CanonicalReportOptions, build_canonical_report from .runner import AttestationIssue, RunBinding, RunnerConfig, run_profile from .scenario_contract import REQUIRED_SCENARIOS from .transaction import PlannedWrite, TransactionConflict, TransactionManager @@ -64,7 +63,7 @@ def _profile(unit: UnitId) -> VerificationProfile: "tests", "test", "pytest", - "checker-owned-config", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ("REQ-1",), (PurePosixPath("tests/test_widget.py"),), ) @@ -131,6 +130,28 @@ def _git(root: Path, *arguments: str) -> str: return completed.stdout.strip() +def _candidate( + arguments: argparse.Namespace, root: Path, *command: str +) -> subprocess.CompletedProcess[str]: + """Run one public command from the separately installed candidate wheel.""" + return subprocess.run( + [arguments.candidate_python, "-I", "-m", "pdd.cli", *command], + cwd=root, + capture_output=True, + text=True, + check=False, + env={ + key: value + for key, value in os.environ.items() + if not any( + marker in key.upper() + for marker in ("API_KEY", "CREDENTIAL", "PASSWORD", "SECRET", "TOKEN") + ) + and key not in {"PYTHONPATH", "PYTHONHOME", "PDD_PATH"} + }, + ) + + def _git_fixture(root: Path) -> None: _git(root, "init", "-q") _git(root, "config", "user.email", "checker@example.com") @@ -372,8 +393,20 @@ def _evidence_guards(_arguments: argparse.Namespace) -> ScenarioResult: ) policy = AttestationTrustPolicy({"checker-fixture": signer.public_key_bytes()}) policy.verify(envelope, binding, now=now) + policy.verify(envelope, binding, now=now) + replayed = signer.issue( + AttestationRequest( + "rebound-attestation", + binding, + envelope.results, + "nonce", + now, + ) + ) + forged = replace(envelope, signature="Zm9yZ2Vk") for candidate, expected in ( - (envelope, binding), + (replayed, binding), + (forged, binding), ( signer.issue( AttestationRequest( @@ -555,12 +588,138 @@ def _transactional_report(_arguments: argparse.Namespace) -> ScenarioResult: ) -def _built_wheel(_arguments: argparse.Namespace) -> ScenarioResult: +def _built_wheel(arguments: argparse.Namespace) -> ScenarioResult: if "site-packages" not in Path(__file__).resolve().parts: raise AssertionError("scenario harness is not running from an installed wheel") + probe = subprocess.run( + [ + arguments.candidate_python, + "-I", + "-c", + "import pathlib,pdd; print(pathlib.Path(pdd.__file__).resolve())", + ], + capture_output=True, + text=True, + check=False, + ) + candidate_environment = Path(arguments.candidate_python).absolute().parents[1].resolve() + try: + Path(probe.stdout.strip()).resolve().relative_to(candidate_environment) + except ValueError as exc: + raise AssertionError("candidate package was not imported from its wheel venv") from exc + if probe.returncode != 0 or "site-packages" not in probe.stdout: + raise AssertionError("candidate wheel import probe failed") return ScenarioResult("built-wheel-clean-environment", "PASS") +def _candidate_public_report(arguments: argparse.Namespace) -> ScenarioResult: + with tempfile.TemporaryDirectory(prefix="pdd-candidate-report-") as directory: + root = Path(directory) + _git_fixture(root) + initialize_repository_identity( + root, "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0" + ) + (root / "prompts").mkdir() + (root / "src").mkdir() + (root / "tests").mkdir() + (root / "prompts/widget_python.prompt").write_text("REQ-1: widget\n") + (root / "src/widget.py").write_text("value = 1\n") + (root / "tests/test_widget.py").write_text("def test_widget(): assert True\n") + (root / "architecture.json").write_text( + '[{"filename":"widget_python.prompt","filepath":"src/widget.py"}]\n' + ) + (root / ".pdd/verification-profiles.json").write_text( + json.dumps( + { + "profiles": [ + { + "prompt_path": "prompts/widget_python.prompt", + "language_id": "python", + "required_requirement_ids": ["REQ-1"], + "obligations": [ + { + "obligation_id": "pytest", + "kind": "test", + "validator_id": "pytest", + "validator_config_digest": ( + "e3b0c44298fc1c149afbf4c8996fb924" + "27ae41e4649b934ca495991b7852b855" + ), + "requirement_ids": ["REQ-1"], + "artifact_paths": ["tests/test_widget.py"], + } + ], + } + ] + } + ) + ) + commit = _commit(root, "candidate report fixture") + output = root / "candidate-report.json" + result = _candidate( + arguments, + root, + "sync", + "certify", + "--base-ref", + commit, + "--head-ref", + commit, + "--replay-ledger", + str(root.parent / "candidate-replay.json"), + "--output", + str(output), + ) + report = json.loads(output.read_text(encoding="utf-8")) + if result.returncode != 1: + raise AssertionError("unbaselined candidate report did not fail closed") + if report.get("counts", {}).get("unbaselined") != 1: + raise AssertionError("candidate report omitted unbaselined managed unit") + return ScenarioResult("candidate-wheel-public-report", "PASS") + + +def _candidate_transaction_recovery(arguments: argparse.Namespace) -> ScenarioResult: + writes = ( + PlannedWrite(PurePosixPath("src/widget.py"), b"new\n", "100644"), + PlannedWrite(PurePosixPath(".pdd/evidence/widget.json"), b"{}\n", "100644"), + ) + with tempfile.TemporaryDirectory(prefix="pdd-candidate-recovery-") as directory: + root = Path(directory) + (root / "src").mkdir() + (root / "src/widget.py").write_text("old\n") + manager = TransactionManager(root) + manager.prepare("candidate-recovery", writes) + + def crash(event: str) -> None: + if event == "after_install:0": + raise SystemExit("injected process death") + + try: + manager.commit("candidate-recovery", crash_hook=crash) + except SystemExit: + pass + result = _candidate( + arguments, + root, + "recover", + "--transaction", + "candidate-recovery", + ) + if result.returncode != 0: + raise AssertionError("candidate public recovery command failed") + json_start = result.stdout.find("{") + if json_start < 0: + raise AssertionError("candidate recovery emitted no machine result") + payload, _end = json.JSONDecoder().raw_decode(result.stdout[json_start:]) + if payload.get("phase") != "COMMITTED": + raise AssertionError("candidate recovery did not commit the journal") + if (root / "src/widget.py").read_text() != "new\n": + raise AssertionError("candidate recovery left old artifact bytes") + if (root / ".pdd/evidence/widget.json").read_text() != "{}\n": + raise AssertionError("candidate recovery left partial evidence state") + return ScenarioResult("candidate-wheel-transaction-recovery", "PASS") + + def _cloud_canary(arguments: argparse.Namespace) -> ScenarioResult: cloud = Path(arguments.cloud_root).resolve() count = count_vendored_sync_semantics( @@ -568,15 +727,34 @@ def _cloud_canary(arguments: argparse.Namespace) -> ScenarioResult: ) if count: raise AssertionError(f"pdd_cloud retains {count} vendored sync semantics") + before = _git(cloud, "status", "--porcelain", "--untracked-files=all") with tempfile.TemporaryDirectory(prefix="pdd-cloud-canary-") as directory: - report = build_canonical_report( + temporary = Path(directory) + output = temporary / "report.json" + candidate = _candidate( + arguments, cloud, - CanonicalReportOptions( - base_ref=arguments.cloud_base_ref, - head_ref=arguments.cloud_head_ref, - replay_ledger_path=Path(directory) / "replay.json", - ), + "sync", + "certify", + "--base-ref", + arguments.cloud_base_ref, + "--head-ref", + arguments.cloud_head_ref, + "--replay-ledger", + str(temporary / "replay.json"), + "--output", + str(output), + ) + report = json.loads(output.read_text(encoding="utf-8")) + after = _git(cloud, "status", "--porcelain", "--untracked-files=all") + if candidate.returncode != 0: + counts = report.get("counts", {}) + raise AssertionError( + "pdd_cloud candidate command is red: " + + json.dumps(counts, sort_keys=True, separators=(",", ":")) ) + if before != after: + raise AssertionError("pdd_cloud candidate canary mutated the checkout") if report.get("ok") is not True: counts = report.get("counts", {}) raise AssertionError( @@ -606,6 +784,8 @@ def _owned_harness(_arguments: argparse.Namespace) -> ScenarioResult: "transactional-canonical-report": _transactional_report, "merge-group-base-movement-and-stale-repair": _merge_base_movement, "built-wheel-clean-environment": _built_wheel, + "candidate-wheel-public-report": _candidate_public_report, + "candidate-wheel-transaction-recovery": _candidate_transaction_recovery, "pdd-cloud-real-consumer-canary": _cloud_canary, "released-checker-owned-scenario-harness": _owned_harness, } @@ -635,6 +815,7 @@ def main() -> None: parser.add_argument("--cloud-root", required=True) parser.add_argument("--cloud-base-ref", required=True) parser.add_argument("--cloud-head-ref", required=True) + parser.add_argument("--candidate-python", required=True) arguments = parser.parse_args() results = run_scenarios(arguments) payload = { diff --git a/pdd/sync_core/snapshot.py b/pdd/sync_core/snapshot.py index f64556861..c26453999 100644 --- a/pdd/sync_core/snapshot.py +++ b/pdd/sync_core/snapshot.py @@ -61,7 +61,7 @@ def add(role: str, relpath: PurePosixPath, required: bool = True) -> None: return UnitSnapshot( unit.unit_id, tuple(sorted(artifacts.values())), - manifest.digest(), + manifest.unit_digest(unit), closure.digest(), profile.profile_digest, closure.has_nondeterministic_query, diff --git a/pdd/sync_core/transaction.py b/pdd/sync_core/transaction.py index 5ff8b304a..3a986893f 100644 --- a/pdd/sync_core/transaction.py +++ b/pdd/sync_core/transaction.py @@ -360,6 +360,22 @@ def _validate_prepared_entries( raise TransactionError(f"prepared transaction blob is unsafe: {relpath}") if _digest(prepared.read_bytes()) != str(item["desired_digest"]): raise TransactionError(f"prepared transaction blob is corrupt: {relpath}") + precondition = item.get("precondition") + if not isinstance(precondition, dict): + raise TransactionError("transaction precondition is malformed") + before = _parse_state(precondition) + rollback_name = item.get("rollback_blob") + if not before.exists: + if rollback_name is not None: + raise TransactionError( + f"unexpected rollback transaction blob: {relpath}" + ) + continue + rollback = transaction_dir / str(rollback_name) + if rollback.is_symlink() or not rollback.is_file(): + raise TransactionError(f"rollback transaction blob is unsafe: {relpath}") + if _digest(rollback.read_bytes()) != before.digest: + raise TransactionError(f"rollback transaction blob is corrupt: {relpath}") def _restore_entries( self, transaction_dir: Path, entries: list[object] diff --git a/pdd/sync_core/trust.py b/pdd/sync_core/trust.py index 229c97cdd..ab1e51248 100644 --- a/pdd/sync_core/trust.py +++ b/pdd/sync_core/trust.py @@ -6,11 +6,12 @@ import json import os import stat +import subprocess import tempfile from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Mapping, Optional +from typing import Mapping, Optional, Protocol from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives import serialization @@ -28,11 +29,23 @@ class AttestationError(ValueError): """Raised when semantic evidence fails protected trust policy.""" +class AttestationIssuer(Protocol): + """Signer surface accepted by the trusted runner.""" + + issuer: str + + def public_key_bytes(self) -> bytes: + """Return the expected Ed25519 public key.""" + + def issue(self, request: "AttestationRequest") -> "AttestationEnvelope": + """Issue one signed attestation envelope.""" + + class ReplayStore: """Interface for atomic cross-verifier attestation nonce consumption.""" def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: - """Record a nonce exactly once or raise AttestationError.""" + """Bind a nonce to one attestation ID, allowing idempotent rechecks.""" raise NotImplementedError def is_durable(self) -> bool: @@ -51,6 +64,8 @@ def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: key = (issuer, nonce) previous = self._seen.get(key) if previous is not None: + if previous == attestation_id: + return raise AttestationError("attestation nonce was replayed") self._seen[key] = attestation_id @@ -114,10 +129,11 @@ def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: records = self._load() key = base64.b64encode(f"{issuer}\0{nonce}".encode()).decode("ascii") previous = records.get(key) - if previous is not None: + if previous is not None and previous != attestation_id: raise AttestationError("attestation nonce was replayed") - records[key] = attestation_id - self._write(records) + if previous is None: + records[key] = attestation_id + self._write(records) if stat.S_IMODE(self.path.stat().st_mode) != 0o600: raise AttestationError("replay ledger permissions are unsafe") @@ -218,7 +234,7 @@ class AttestationRequest: results: tuple[ObligationEvidence, ...] nonce: str issued_at: Optional[datetime] = None - lifetime: timedelta = timedelta(hours=1) + lifetime: timedelta = timedelta(days=8) @dataclass(frozen=True) @@ -299,21 +315,63 @@ def sign(self, envelope: AttestationEnvelope) -> AttestationEnvelope: def issue(self, request: AttestationRequest) -> AttestationEnvelope: """Create a signed envelope bound to one checked commit and base.""" - issued = request.issued_at or datetime.now(timezone.utc) - validity = AttestationValidity( - _timestamp(issued), - _timestamp(issued + request.lifetime), - request.nonce, + return self.sign(_unsigned_envelope(self.issuer, request)) + + +def _unsigned_envelope( + issuer: str, request: AttestationRequest +) -> AttestationEnvelope: + """Build the canonical unsigned statement sent to a signing authority.""" + issued = request.issued_at or datetime.now(timezone.utc) + validity = AttestationValidity( + _timestamp(issued), + _timestamp(issued + request.lifetime), + request.nonce, + ) + return AttestationEnvelope( + request.attestation_id, + issuer, + request.binding, + request.results, + validity, + ) + + +class RemoteAttestationSigner: + """Issue evidence through a protected signer command and verify its response.""" + + def __init__(self, issuer: str, public_key: bytes, command: tuple[str, ...]) -> None: + if not issuer or len(public_key) != 32 or not command: + raise ValueError("remote attestation signer configuration is invalid") + self.issuer = issuer + self._public_key = public_key + self._command = command + + def public_key_bytes(self) -> bytes: + """Return the protected expected public key.""" + return self._public_key + + def issue(self, request: AttestationRequest) -> AttestationEnvelope: + """Send canonical claims to the remote authority and verify its signature.""" + envelope = _unsigned_envelope(self.issuer, request) + result = subprocess.run( + self._command, + input=envelope.payload(), + capture_output=True, + check=False, ) - return self.sign( - AttestationEnvelope( - request.attestation_id, - self.issuer, - request.binding, - request.results, - validity, + if result.returncode != 0: + raise AttestationError("remote attestation signer rejected the request") + try: + signature = base64.b64decode(result.stdout.strip(), validate=True) + Ed25519PublicKey.from_public_bytes(self._public_key).verify( + signature, envelope.payload() ) - ) + except (ValueError, InvalidSignature) as exc: + raise AttestationError( + "remote attestation signer returned an invalid signature" + ) from exc + return replace(envelope, signature=base64.b64encode(signature).decode("ascii")) class AttestationTrustPolicy: @@ -326,7 +384,7 @@ def __init__( revoked_issuers: frozenset[str] = frozenset(), revoked_attestations: frozenset[str] = frozenset(), clock_skew: timedelta = timedelta(minutes=5), - maximum_lifetime: timedelta = timedelta(hours=1), + maximum_lifetime: timedelta = timedelta(days=8), replay_store: ReplayStore | None = None, ) -> None: # pylint: disable=too-many-arguments diff --git a/pdd/sync_core/types.py b/pdd/sync_core/types.py index e494b4798..4fe3586ee 100644 --- a/pdd/sync_core/types.py +++ b/pdd/sync_core/types.py @@ -226,7 +226,18 @@ def complete(self) -> bool: for item in required_obligations for requirement_id in item.requirement_ids } - return set(self.required_requirement_ids).issubset(covered) + required = set(self.required_requirement_ids) + if covered != required: + return False + opaque = {item for item in required if item.startswith("CONTRACT-SHA256:")} + human_covered = { + requirement_id + for item in required_obligations + if item.kind == "human-attestation" + and item.validator_id == "threshold-ed25519" + for requirement_id in item.requirement_ids + } + return opaque.issubset(human_covered) @dataclass(frozen=True, order=True) diff --git a/pdd/sync_core/verification.py b/pdd/sync_core/verification.py index 36e8f1b09..159d2c61e 100644 --- a/pdd/sync_core/verification.py +++ b/pdd/sync_core/verification.py @@ -190,6 +190,11 @@ def _effective_profile( head: _ProfileInput | None, ) -> tuple[VerificationProfile, list[str]]: invalid: list[str] = [] + if base is None and head is not None: + invalid.append( + f"{unit_id.prompt_relpath}: candidate-only profile lacks protected approval" + ) + head = None base_requirements = set(base.requirements if base else ()) head_requirements = set(head.requirements if head else ()) if base_requirements - head_requirements: diff --git a/pdd/sync_determine_operation.py b/pdd/sync_determine_operation.py index 99e0e35a2..75a4ec51a 100644 --- a/pdd/sync_determine_operation.py +++ b/pdd/sync_determine_operation.py @@ -7,14 +7,13 @@ """ import os -import re import sys import glob import json import hashlib import subprocess import fnmatch -from pathlib import Path +from pathlib import Path, PurePosixPath from dataclasses import dataclass, field from typing import Dict, List, Optional, Any, Tuple from datetime import datetime @@ -45,7 +44,6 @@ ) from pdd.load_prompt_template import load_prompt_template from pdd.llm_invoke import llm_invoke -from pdd.get_language import get_language from pdd.template_expander import expand_template from pdd.architecture_registry import extract_modules from pdd.sync_order import extract_module_from_include @@ -1728,58 +1726,57 @@ def calculate_sha256(file_path: Path) -> Optional[str]: return None -_INCLUDE_PATTERN = re.compile(r']*>(.*?)') -_BACKTICK_INCLUDE_PATTERN = re.compile(r'```<([^>]*?)>```') - - -def _resolve_include_path(include_ref: str, prompt_dir: Path) -> Optional[Path]: - """Resolve an reference to an absolute Path.""" - p = Path(include_ref) - if p.is_absolute() and p.exists(): - return p - candidate = prompt_dir / include_ref - if candidate.exists(): - return candidate - candidate = Path.cwd() / include_ref - if candidate.exists(): - return candidate - return None - - def extract_include_deps(prompt_path: Path) -> Dict[str, str]: """Extract include dependency paths and their hashes from a prompt file. Returns a dict mapping resolved dependency paths to their SHA256 hashes. Only includes dependencies that exist on disk. """ - if not prompt_path.exists(): - return {} - - try: - prompt_content = prompt_path.read_text(encoding='utf-8', errors='ignore') - except (IOError, OSError): - return {} - - include_refs = _INCLUDE_PATTERN.findall(prompt_content) - include_refs += _BACKTICK_INCLUDE_PATTERN.findall(prompt_content) - - if not include_refs: - return {} + from pdd.continuous_sync import canonical_sync_enabled, repository_root + from pdd.sync_core.includes import ( + IncludeGraphError, + build_include_closure, + parse_include_references, + ) + from pdd.sync_core.path_policy import PathPolicy, PathPolicyError - deps = {} - prompt_dir = prompt_path.parent - for ref in sorted(set(r.strip() for r in include_refs)): - dep_path = _resolve_include_path(ref, prompt_dir) - if dep_path and dep_path.exists(): - dep_hash = calculate_sha256(dep_path) - if dep_hash: + if not canonical_sync_enabled(prompt_path): + if not prompt_path.exists(): + return {} + try: + content = prompt_path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return {} + dependencies: Dict[str, str] = {} + for reference in parse_include_references(content): + declared = Path(reference.path) + candidates = ( + (declared,) + if declared.is_absolute() + else (prompt_path.parent / declared, Path.cwd() / declared) + ) + dependency = next((item for item in candidates if item.is_file()), None) + if dependency is None: + continue + digest = calculate_sha256(dependency) + if digest: try: - rel_path = dep_path.relative_to(Path.cwd()) + key = str(dependency.relative_to(Path.cwd())) except ValueError: - rel_path = dep_path - deps[str(rel_path)] = dep_hash - - return deps + key = str(dependency) + dependencies[key] = digest + return dependencies + canonical_root = repository_root(prompt_path) + if not prompt_path.exists(): + raise FileNotFoundError(prompt_path) + try: + prompt_relpath = prompt_path.resolve().relative_to(canonical_root) + closure = build_include_closure( + PurePosixPath(prompt_relpath.as_posix()), PathPolicy(canonical_root) + ) + except (ValueError, IncludeGraphError, PathPolicyError) as exc: + raise ValueError(f"canonical include closure failed: {prompt_path}") from exc + return {item.relpath.as_posix(): item.digest for item in closure.artifacts} def calculate_prompt_hash(prompt_path: Path, stored_deps: Optional[Dict[str, str]] = None) -> Optional[str]: @@ -1801,48 +1798,33 @@ def calculate_prompt_hash(prompt_path: Path, stored_deps: Optional[Dict[str, str return None try: - prompt_content = prompt_path.read_text(encoding='utf-8', errors='ignore') - except (IOError, OSError): + prompt_content = prompt_path.read_text(encoding="utf-8") + except (IOError, OSError, UnicodeDecodeError): return None + from pdd.sync_core.includes import parse_include_references - # Try to find include refs in current prompt content - include_refs = _INCLUDE_PATTERN.findall(prompt_content) - include_refs += _BACKTICK_INCLUDE_PATTERN.findall(prompt_content) - - # Resolve to actual paths - prompt_dir = prompt_path.parent - dep_paths = [] - if include_refs: - for ref in sorted(set(r.strip() for r in include_refs)): - dep_path = _resolve_include_path(ref, prompt_dir) - if dep_path and dep_path.exists(): - dep_paths.append(dep_path) - elif stored_deps: - # No include tags in prompt — use stored dependency paths from fingerprint - for dep_path_str in sorted(stored_deps.keys()): - dep_path = Path(dep_path_str) - if dep_path.exists(): - dep_paths.append(dep_path) - - if not dep_paths: - return calculate_sha256(prompt_path) - - # Build composite hash: prompt bytes + sorted dependency contents - hasher = hashlib.sha256() - try: - with open(prompt_path, 'rb') as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - except (IOError, OSError): - return None - - for dep_path in dep_paths: + references = parse_include_references(prompt_content) + if references: try: - with open(dep_path, 'rb') as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - except (IOError, OSError): - pass + dependencies = extract_include_deps(prompt_path) + except (FileNotFoundError, ValueError): + return None + else: + dependencies = dict(stored_deps or {}) + for path, expected_digest in dependencies.items(): + candidate = Path(path) + if not candidate.is_absolute(): + candidate = Path.cwd() / candidate + if calculate_sha256(candidate) != expected_digest: + if not candidate.is_file(): + return None + dependencies[path] = calculate_sha256(candidate) or "" + + hasher = hashlib.sha256() + hasher.update(prompt_path.read_bytes()) + for path, digest in sorted(dependencies.items()): + hasher.update(path.encode("utf-8")) + hasher.update(digest.encode("ascii")) return hasher.hexdigest() diff --git a/pdd/sync_orchestration.py b/pdd/sync_orchestration.py index 4e7e70c14..f0f66a3ef 100644 --- a/pdd/sync_orchestration.py +++ b/pdd/sync_orchestration.py @@ -11,7 +11,7 @@ import subprocess import re import os -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Dict, Any, Optional, List, Callable from dataclasses import asdict, dataclass, field import tempfile @@ -682,6 +682,34 @@ def _save_fingerprint_atomic(basename: str, language: str, operation: str, include_deps_override: Pre-captured include deps (Issue #522). Used when auto-deps may have stripped tags before fingerprint save. """ + from .continuous_sync import canonical_sync_enabled, repository_root + + start = Path(paths.get("prompt") or Path.cwd()) + if canonical_sync_enabled(start): + from .sync_core import ( + attestation_signer_from_environment, + finalize_unit, + ) + + root = repository_root(start) + protected_base = os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") + if not protected_base: + raise RuntimeError( + "canonical sync requires PDD_SYNC_PROTECTED_BASE_SHA" + ) + prompt_path = Path(paths["prompt"]).resolve() + try: + module = PurePosixPath(prompt_path.relative_to(root).as_posix()) + except ValueError as exc: + raise RuntimeError("canonical prompt path escapes project root") from exc + finalize_unit( + root, + module, + base_ref=protected_base, + head_ref="HEAD", + signer=attestation_signer_from_environment(), + ) + return if atomic_state: # Buffer for atomic write from datetime import datetime, timezone @@ -2645,6 +2673,25 @@ def sync_worker_logic(): test_output_excerpt: Optional[str] = None operation_rollback = None + from .continuous_sync import canonical_sync_enabled, project_root + + operation_root = project_root(pdd_files.get("prompt") or Path.cwd()) + if canonical_sync_enabled(operation_root): + rollback_paths = [] + for value in pdd_files.values(): + if isinstance(value, Path): + rollback_paths.append(value) + elif isinstance(value, list): + rollback_paths.extend( + item for item in value if isinstance(item, Path) + ) + for shared in ( + operation_root / "architecture.json", + operation_root / "project_dependencies.csv", + ): + rollback_paths.append(shared) + operation_rollback = OperationFileRollback(rollback_paths) + # Issue #159 fix: Use atomic state for consistent run_report + fingerprint writes set_current_operation(operation) # Drop any stale LLM trace for this operation key so failure paths only @@ -2656,10 +2703,11 @@ def sync_worker_logic(): try: if operation == 'auto-deps': temp_output = Path(str(pdd_files['prompt']).replace('.prompt', '_with_deps.prompt')) - operation_rollback = _build_auto_deps_rollback( - pdd_files['prompt'], - temp_output, - ) + if operation_rollback is None: + operation_rollback = _build_auto_deps_rollback( + pdd_files['prompt'], + temp_output, + ) original_content = pdd_files['prompt'].read_text(encoding='utf-8') # Issue #522: Capture include deps BEFORE auto-deps may strip tags from .sync_determine_operation import extract_include_deps diff --git a/pdd/sync_order.py b/pdd/sync_order.py index 0949b99f3..ecee242a0 100644 --- a/pdd/sync_order.py +++ b/pdd/sync_order.py @@ -14,6 +14,7 @@ from rich.console import Console from pdd.construct_paths import _is_known_language +from pdd.sync_core.includes import include_paths # Initialize rich console console = Console() @@ -44,71 +45,7 @@ def extract_includes_from_file(file_path: Path) -> Set[str]: try: content = file_path.read_text(encoding="utf-8") - includes: Set[str] = set() - - # Reviewer 5th-pass (F1) + 6th-pass (F3): mirror the preprocessor's - # full tag tolerance. ``pdd.preprocess.process_include_tags`` accepts - # three forms: - # - bare body: ``path`` - # - attributed body: ``path`` - # - self-closing: ```` - # Real prompts use all three; missing any one means a real PDD - # include form sits outside #739's safety net. - # Body-form must exclude self-closing ``. Without - # the `(?` lookbehind, the body-form regex would absorb the - # entire `\nfoo.md` span as one - # match, losing the inner include and producing garbage. - # - # Capture the attribute string so we can honor a `path="..."` - # attribute the same way ``pdd.preprocess.process_include_tags`` - # does: ``attrs.get('path') or content.strip()`` (preprocess.py - # line ~501). Without this, an attributed body include such as - # ``fallback.md`` would - # be reported as ``fallback.md`` here while the preprocessor - # actually resolves it as ``docs/source.md`` — and the scope - # guard's allowlist would diverge from the real include graph. - single_matches = re.findall( - r']*?))?(?(.*?)', content, re.DOTALL - ) - for attrs, body in single_matches: - path_value: Optional[str] = None - if attrs: - attr_match = re.search( - r'path\s*=\s*["\']([^"\']+)["\']', attrs - ) - if attr_match: - path_value = attr_match.group(1).strip() - if not path_value: - path_value = body.strip() - if path_value: - includes.add(path_value) - - # Self-closing form: extract the ``path="..."`` attribute value. - self_closing_matches = re.findall( - r']*?)\s*/>', content - ) - for attrs in self_closing_matches: - path_match = re.search( - r'path\s*=\s*["\']([^"\']+)["\']', attrs - ) - if path_match: - stripped = path_match.group(1).strip() - if stripped: - includes.add(stripped) - - many_matches = re.findall( - r']*?)?>(.*?)', content, re.DOTALL - ) - for m in many_matches: - # Mirror pdd.preprocess.process_include_many_tags: split on - # newlines first, then on commas, then strip and drop empties. - for part in m.splitlines(): - for item in part.split(","): - stripped = item.strip() - if stripped: - includes.add(stripped) - - return includes + return include_paths(content) except Exception as e: logger.error(f"Error reading file {file_path}: {e}") return set() diff --git a/pdd/update_main.py b/pdd/update_main.py index 0543cbb2f..1b1c704a6 100644 --- a/pdd/update_main.py +++ b/pdd/update_main.py @@ -1199,6 +1199,9 @@ def _finalize_single_file_fingerprint( model=model, ) except Exception as exc: + from .sync_core.finalize import CanonicalFinalizationError + if isinstance(exc, CanonicalFinalizationError): + raise if not quiet: rprint(f"[warning][metadata] Fingerprint save failed: {exc}[/warning]") @@ -1462,7 +1465,10 @@ def update_main( cost=result.get("cost", 0.0), model=result.get("model", "unknown"), ) - except Exception: + except Exception as exc: + from .sync_core.finalize import CanonicalFinalizationError + if isinstance(exc, CanonicalFinalizationError): + raise pass # Best-effort; don't fail the update else: if "Success" in result.get("status", ""): diff --git a/tests/e2e/test_issue_1932_continuous_sync_guarantee.py b/tests/e2e/test_issue_1932_continuous_sync_guarantee.py index d9f917f26..be77a352f 100644 --- a/tests/e2e/test_issue_1932_continuous_sync_guarantee.py +++ b/tests/e2e/test_issue_1932_continuous_sync_guarantee.py @@ -104,7 +104,7 @@ def pdd_project(tmp_path: Path) -> Path: encoding="utf-8", ) (repo / "prompts/widget_python.prompt").write_text( - "Build a deterministic widget.\n../docs/widget.md\n", + "Build a deterministic widget.\ndocs/widget.md\n", encoding="utf-8", ) (repo / "prompts/runtime_template_LLM.prompt").write_text( @@ -144,7 +144,7 @@ def pdd_project(tmp_path: Path) -> Path: def _unit_classification(report: dict) -> str: - units = report["units"] + units = [unit for unit in report["units"] if unit["basename"] == "widget"] assert len(units) == 1 return units[0]["classification"] @@ -171,31 +171,35 @@ def _last_ledger_classification(repo: Path) -> str: assert ledger.exists() lines = [line for line in ledger.read_text(encoding="utf-8").splitlines() if line] assert lines - return json.loads(lines[-1])["classification"] + widget_entries = [ + json.loads(line) for line in lines if json.loads(line)["basename"] == "widget" + ] + assert widget_entries + return widget_entries[-1]["classification"] def _fingerprint(repo: Path) -> dict: return json.loads((repo / ".pdd/meta/widget_python.json").read_text(encoding="utf-8")) -def _assert_valid_fingerprint(repo: Path) -> None: - fingerprint = _fingerprint(repo) - for key in ("prompt_hash", "code_hash", "example_hash", "test_hash"): - assert fingerprint.get(key), f"{key} missing from fingerprint" - - -def test_issue_1932_clean_baseline_reports_no_drift(pdd_project: Path) -> None: - reconcile = _pdd_json(pdd_project, "reconcile", "--json", "--strict") +def test_issue_1932_complete_inventory_blocks_unbaselined_units(pdd_project: Path) -> None: + reconcile = _pdd_json( + pdd_project, "reconcile", "--json", "--strict", check=False + ) sync = _pdd_json(pdd_project, "sync", "--dry-run", "--json") update = _pdd_json(pdd_project, "update", "--all", "--dry-run", "--json") for report in (reconcile, sync, update): - assert report["ok"] is True + assert report["ok"] is False assert report["metadata_stale"] == 0 assert report["summary"]["conflicts"] == 0 - assert report["summary"]["unbaselined"] == 0 - assert report["summary"]["total"] == 1 - assert report["units"][0]["basename"] == "widget" + assert report["summary"]["unbaselined"] == 2 + assert report["summary"]["total"] == 3 + assert {unit["basename"] for unit in report["units"]} == { + "widget", + "runtime_template", + "unbaselined_helper", + } @pytest.mark.parametrize( @@ -212,7 +216,7 @@ def test_issue_1932_clean_baseline_reports_no_drift(pdd_project: Path) -> None: "prompts/widget_python.prompt", lambda p: p.write_text( "Build a deterministic widget with a new requirement.\n" - "../docs/widget.md\n", + "docs/widget.md\n", encoding="utf-8", ), "PROMPT_CHANGED", @@ -234,14 +238,13 @@ def test_issue_1932_clean_baseline_reports_no_drift(pdd_project: Path) -> None: ), ], ) -def test_issue_1932_all_consumers_classify_and_heal_idempotently( +def test_issue_1932_all_consumers_classify_without_blind_healing( pdd_project: Path, name: str, path: str, edit: Callable[[Path], None], classification: str, ) -> None: - baseline = _git(pdd_project, "rev-parse", "HEAD").stdout.strip() edit(pdd_project / path) _git(pdd_project, "add", path) _git(pdd_project, "commit", "-q", "-m", f"{name} drift") @@ -249,20 +252,19 @@ def test_issue_1932_all_consumers_classify_and_heal_idempotently( _assert_all_consumers_classify(pdd_project, classification) assert _last_ledger_classification(pdd_project) == classification - first_heal = _pdd_json(pdd_project, "reconcile", "--json", "--strict", "--heal") - assert first_heal["ok"] is True - assert first_heal["metadata_stale"] == 0 - _assert_valid_fingerprint(pdd_project) - - _git(pdd_project, "add", ".pdd/meta/widget_python.json") - _git(pdd_project, "commit", "-q", "-m", f"{name} metadata heal") - - second_heal = _pdd_json(pdd_project, "reconcile", "--json", "--strict", "--heal") - assert second_heal["ok"] is True - assert second_heal["metadata_stale"] == 0 - _git(pdd_project, "diff", "--exit-code") - - _git(pdd_project, "reset", "--hard", baseline) + fingerprint_before = (pdd_project / ".pdd/meta/widget_python.json").read_text( + encoding="utf-8" + ) + refused = _run( + pdd_project, + [sys.executable, "-m", "pdd", "reconcile", "--json", "--heal"], + check=False, + ) + assert refused.returncode != 0 + assert "--heal is disabled" in refused.stderr + assert (pdd_project / ".pdd/meta/widget_python.json").read_text( + encoding="utf-8" + ) == fingerprint_before def test_issue_1932_unbaselined_fingerprints_are_not_stamped(pdd_project: Path) -> None: @@ -275,7 +277,6 @@ def test_issue_1932_unbaselined_fingerprints_are_not_stamped(pdd_project: Path) "reconcile", "--json", "--strict", - "--heal", "--module", "widget", check=False, @@ -290,7 +291,6 @@ def test_issue_1932_unbaselined_fingerprints_are_not_stamped(pdd_project: Path) "reconcile", "--json", "--strict", - "--heal", "--module", "widget", check=False, @@ -307,7 +307,7 @@ def test_issue_1932_prompt_code_coedit_is_conflict_without_data_loss(pdd_project prompt_path = pdd_project / "prompts/widget_python.prompt" prompt_text = ( "Build a deterministic widget with a conflicting prompt change.\n" - "../docs/widget.md\n" + "docs/widget.md\n" ) prompt_path.write_text(prompt_text, encoding="utf-8") (pdd_project / "src/widget.py").write_text( @@ -316,14 +316,7 @@ def test_issue_1932_prompt_code_coedit_is_conflict_without_data_loss(pdd_project ) _assert_all_consumers_classify(pdd_project, "CONFLICT") - conflict = _pdd_json( - pdd_project, - "reconcile", - "--json", - "--strict", - "--heal", - check=False, - ) + conflict = _pdd_json(pdd_project, "reconcile", "--json", "--strict", check=False) assert conflict["ok"] is False assert conflict["conflicts"][0]["operation"] == "conflict" assert (pdd_project / ".pdd/meta/widget_python.json").read_text(encoding="utf-8") == fingerprint_before @@ -355,15 +348,12 @@ def test_issue_1932_deleted_generated_artifact_is_failure_not_in_sync( assert report["failures"][0]["failure"] == "missing_artifacts" assert report["failures"][0]["changed_files"] == ["code"] - healed = _pdd_json( + refused = _run( pdd_project, - "reconcile", - "--json", - "--strict", - "--heal", + [sys.executable, "-m", "pdd", "reconcile", "--json", "--heal"], check=False, ) - assert healed["ok"] is False + assert refused.returncode != 0 assert (pdd_project / ".pdd/meta/widget_python.json").read_text(encoding="utf-8") == fingerprint_before @@ -406,7 +396,7 @@ def test_issue_1932_deleted_prompt_stays_discovered_from_metadata( check=False, ) assert default_report["ok"] is False - assert default_report["summary"]["total"] == 1 + assert default_report["summary"]["total"] == 3 assert default_report["failures"][0]["failure"] == "missing_artifacts" assert default_report["failures"][0]["changed_files"] == ["prompt"] diff --git a/tests/test_ci_drift_heal.py b/tests/test_ci_drift_heal.py index 13b07a5fa..d35e86f96 100644 --- a/tests/test_ci_drift_heal.py +++ b/tests/test_ci_drift_heal.py @@ -243,20 +243,19 @@ def test_module_filter_detects_code_without_prompt(self): mock_sync.assert_not_called() def test_infer_identity_error_skips_module(self): - """Modules that fail identity inference are skipped gracefully.""" + """Modules that fail identity inference fail closed.""" mock_files = [Path("prompts/bad_python.prompt")] with patch("pdd.user_story_tests.discover_prompt_files", return_value=mock_files), \ patch("pdd.operation_log.infer_module_identity", side_effect=ValueError("bad")), \ patch("pdd.sync_determine_operation.sync_determine_operation") as mock_sync: - prompt_drifts, example_drifts = detect_drift() + with pytest.raises(RuntimeError, match="module identity failed"): + detect_drift() - assert len(prompt_drifts) == 0 - assert len(example_drifts) == 0 mock_sync.assert_not_called() - def test_sync_determine_error_skips_module(self): - """Modules that fail sync_determine_operation are skipped.""" + def test_sync_determine_error_fails_closed(self): + """Modules that fail sync_determine_operation block CI.""" mock_files = [Path("prompts/mod_python.prompt")] def fake_infer(path): @@ -265,10 +264,8 @@ def fake_infer(path): with patch("pdd.user_story_tests.discover_prompt_files", return_value=mock_files), \ patch("pdd.operation_log.infer_module_identity", side_effect=fake_infer), \ patch("pdd.sync_determine_operation.sync_determine_operation", side_effect=RuntimeError("fail")): - prompt_drifts, example_drifts = detect_drift() - - assert len(prompt_drifts) == 0 - assert len(example_drifts) == 0 + with pytest.raises(RuntimeError, match="classification failed"): + detect_drift() # --------------------------------------------------------------------------- diff --git a/tests/test_get_language.py b/tests/test_get_language.py index e2d79e0aa..133ecaaab 100644 --- a/tests/test_get_language.py +++ b/tests/test_get_language.py @@ -35,36 +35,35 @@ def test_get_language_without_dot(mock_environment, mock_csv_file): assert get_language('java') == 'Java' def test_get_language_prompt(mock_environment, mock_csv_file): - """Tests get_language with a valid extension without the dot.""" - assert get_language('prompt') == 'LLM' + """Ambiguous extensions require an explicit language elsewhere.""" + assert get_language('prompt') == '' def test_get_language_case_insensitive(mock_environment, mock_csv_file): """Tests get_language for case insensitivity.""" assert get_language('.PY') == 'Python' def test_get_language_not_found(mock_environment, mock_csv_file): - """Tests get_language with an extension not in the CSV.""" - assert get_language('.cpp') == '' + """Bundled registry support is independent of candidate CSV contents.""" + assert get_language('.cpp') == 'C++' def test_get_language_empty_extension(mock_environment, mock_csv_file): """Tests get_language with an empty extension.""" assert get_language('') == '' def test_get_language_missing_environment_variable(monkeypatch): - """Tests get_language when the PDD_PATH environment variable is not set.""" + """Bundled language policy does not require PDD_PATH.""" monkeypatch.delenv("PDD_PATH", raising=False) - with pytest.raises(ValueError, match="PDD_PATH environment variable is not set"): - get_language('.py') + assert get_language('.py') == 'Python' def test_get_language_file_not_found(mock_environment, tmp_path): - """Tests get_language when the CSV file is not found.""" - assert get_language('.py') == '' + """A missing candidate CSV cannot alter bundled policy.""" + assert get_language('.py') == 'Python' def test_get_language_csv_error(mock_environment, mock_csv_file): - """Tests get_language when there's an error reading the CSV file.""" + """A malformed candidate CSV cannot alter bundled policy.""" # Corrupt the CSV file mock_csv_file.write_text("invalid,csv,data") - assert get_language('.py') == '' + assert get_language('.py') == 'Python' # --- Tests for language_format.csv structure and completeness --- @@ -146,4 +145,4 @@ def test_js_ts_have_explicit_test_commands(self, language_format_path): ts_cmd = rows.get('TypeScript', {}).get('run_test_command', '').strip() assert js_cmd == 'node {file}', f"JavaScript run_test_command should be 'node {{file}}', got: {js_cmd}" - assert ts_cmd == 'npx tsx {file}', f"TypeScript run_test_command should be 'npx tsx {{file}}', got: {ts_cmd}" \ No newline at end of file + assert ts_cmd == 'npx tsx {file}', f"TypeScript run_test_command should be 'npx tsx {{file}}', got: {ts_cmd}" diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py index 138b548fb..1f873ee07 100644 --- a/tests/test_preprocess.py +++ b/tests/test_preprocess.py @@ -772,8 +772,8 @@ def test_get_file_path() -> None: # Test with absolute path abs_path = "/absolute/path/test.txt" - path = get_file_path(abs_path) - assert path == abs_path + with pytest.raises(ValueError, match="absolute include path"): + get_file_path(abs_path) # Test for nested XML tags def test_nested_xml_tags() -> None: @@ -1716,7 +1716,7 @@ def test_issue_74_mixed_required_and_optional_variables(): assert len(matches) == 0, f"Found single-brace variables: {matches}" -def test_get_file_path_repo_root_fallback(monkeypatch, tmp_path): +def test_get_file_path_does_not_fallback_outside_active_project(monkeypatch, tmp_path): """ Verifies that get_file_path correctly falls back to the repository root when run from a worktree where import shadowing occurs. @@ -1762,11 +1762,11 @@ def test_get_file_path_repo_root_fallback(monkeypatch, tmp_path): # This is crucial for simulating the 'package_root' calculation in get_default_resolver() monkeypatch.setattr('pdd.path_resolution.__file__', str(mock_path_resolution_file)) - # Expectation: get_file_path should find the file in the mock_repo_root + # Protected path policy keeps resolution rooted at the active project. found_path = get_file_path(mock_file_name) - # Assert that the found path is the one in the mock repository root - assert found_path == str(expected_file_path) + assert found_path == f"./{mock_file_name}" + assert found_path != str(expected_file_path) # ============================================================================ @@ -3390,9 +3390,10 @@ def test_include_many_optional_attribute_silences_missing(tmp_path, monkeypatch, def test_get_file_path_absolute_returned_unchanged(tmp_path) -> None: - """Absolute paths should not be prefixed with ./""" + """Absolute managed include paths are rejected by protected policy.""" p = str(tmp_path / "abs.txt") - assert get_file_path(p) == p + with pytest.raises(ValueError, match="absolute include path"): + get_file_path(p) def test_web_tag_inside_code_block_not_processed() -> None: @@ -3422,4 +3423,4 @@ def test_xml_include_with_lines_attribute(tmp_path, monkeypatch) -> None: result = preprocess(prompt, recursive=False, double_curly_brackets=False) selectors = MockCS.return_value.select.call_args.kwargs["selectors"] assert "lines:2-3" in selectors - assert result == "b\nc" \ No newline at end of file + assert result == "b\nc" diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 08c27418d..512767f3a 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -1,5 +1,6 @@ """Tests for the signed cross-repository certificate predicate.""" +import base64 import json import subprocess from datetime import datetime, timedelta, timezone @@ -11,9 +12,11 @@ CheckerIdentity, GlobalCertificateOptions, LifecycleResult, + NightlyObservation, RepositoryTarget, build_global_certificate, count_vendored_sync_semantics, + signer_from_environment, verify_global_certificate, ) from pdd.sync_core.lifecycle import _isolated_lifecycle_environment @@ -25,6 +28,7 @@ "refs/tags/v-test-checker", "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v-test-checker", ) +OBSERVATION = NightlyObservation(True, True, 0, True, "BLOCKED", 0) @pytest.fixture(autouse=True) @@ -66,7 +70,7 @@ def _report(name): } -def _nightly(path, signer, count=7): +def _nightly(path, signer, count=7, *, include_observation=True): start = datetime(2026, 7, 3, tzinfo=timezone.utc) rows = [] for index in range(count): @@ -94,6 +98,7 @@ def _nightly(path, signer, count=7): "required_nightly_streak": 7, "verification_profile_coverage": 100, "trusted_current_evidence_coverage": 100, + "nightly_observation_complete": 1, } lifecycle = { "lifecycle_matrix_failed": 0, @@ -114,6 +119,8 @@ def _nightly(path, signer, count=7): "scan_ok": True, "ok": index >= 7, } + if include_observation: + body["nightly_observation"] = OBSERVATION.payload() rows.append( { **body, @@ -151,6 +158,38 @@ def test_lifecycle_child_environment_excludes_all_signing_material( assert environment["HOME"] == str(tmp_path / "isolated-home") +def test_certificate_signing_uses_remote_verified_authority( + monkeypatch, +) -> None: + authority = AttestationSigner("certificate-kms", b"k" * 32) + monkeypatch.setenv("PDD_CERTIFICATE_ISSUER", authority.issuer) + monkeypatch.setenv( + "PDD_CERTIFICATE_PUBLIC_KEY", + base64.b64encode(authority.public_key_bytes()).decode("ascii"), + ) + monkeypatch.setenv( + "PDD_CERTIFICATE_SIGNER_COMMAND", json.dumps(["protected-kms-sign"]) + ) + + def remote_sign(command, *, input, capture_output, check): + assert command == ("protected-kms-sign",) + assert capture_output is True and check is False + signature = authority.sign_bytes(input).encode("ascii") + return subprocess.CompletedProcess(command, 0, stdout=signature, stderr=b"") + + monkeypatch.setattr("pdd.sync_core.certificate.subprocess.run", remote_sign) + signer = signer_from_environment() + assert signer.sign_bytes(b"canonical certificate") == authority.sign_bytes( + b"canonical certificate" + ) + + +def test_local_certificate_private_key_is_forbidden(monkeypatch) -> None: + monkeypatch.setenv("PDD_CERTIFICATE_SIGNING_KEY", "candidate-secret") + with pytest.raises(ValueError, match="local certificate signing keys are forbidden"): + signer_from_environment() + + def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatch) -> None: pdd = tmp_path / "pdd" cloud = tmp_path / "pdd_cloud" @@ -176,7 +215,11 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc RepositoryTarget("pdd_cloud", cloud), ), GlobalCertificateOptions( - tmp_path / "replay", lifecycle, nightly, checker_identity=CHECKER + tmp_path / "replay", + lifecycle, + nightly, + checker_identity=CHECKER, + nightly_observation=OBSERVATION, ), signer=signer, ) @@ -230,7 +273,11 @@ def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> RepositoryTarget("pdd_cloud", tmp_path / "pdd_cloud"), ), GlobalCertificateOptions( - tmp_path / "replay", lifecycle, nightly, checker_identity=CHECKER + tmp_path / "replay", + lifecycle, + nightly, + checker_identity=CHECKER, + nightly_observation=OBSERVATION, ), signer=signer, ) @@ -278,6 +325,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( LifecycleResult(0, 0, 0, 0, 0, 0), nightly, checker_identity=CHECKER, + nightly_observation=OBSERVATION, ), signer=signer, ) @@ -422,6 +470,7 @@ def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( LifecycleResult(0, 0, 0, 0, 0, 0), nightly, checker_identity=CHECKER, + nightly_observation=OBSERVATION, ), signer=AttestationSigner("certificate-ci", b"g" * 32), ) @@ -430,6 +479,42 @@ def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( assert certificate["ok"] is False +def test_signed_nightly_rows_without_observations_do_not_extend_streak( + tmp_path, monkeypatch +) -> None: + for name in ("pdd", "pdd_cloud"): + (tmp_path / name).mkdir() + signer = AttestationSigner("certificate-ci", b"g" * 32) + nightly = tmp_path / "nightly.jsonl" + _nightly(nightly, signer, include_observation=False) + monkeypatch.setattr( + "pdd.sync_core.certificate.build_canonical_report", + lambda root, _options: _report(str(root)), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._validate_target", lambda target: target + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 + ) + certificate = build_global_certificate( + ( + RepositoryTarget("pdd", tmp_path / "pdd"), + RepositoryTarget("pdd_cloud", tmp_path / "pdd_cloud"), + ), + GlobalCertificateOptions( + tmp_path / "replay", + LifecycleResult(0, 0, 0, 0, 0, 0), + nightly, + checker_identity=CHECKER, + nightly_observation=OBSERVATION, + ), + signer=signer, + ) + assert certificate["counts"]["nightly_streak"] == 0 + assert certificate["ok"] is False + + def test_nightly_lineage_requires_identity_and_ancestry(tmp_path) -> None: targets = [] reports = [] diff --git a/tests/test_sync_core_manifest.py b/tests/test_sync_core_manifest.py index 80bf17ad5..dbdfd08f0 100644 --- a/tests/test_sync_core_manifest.py +++ b/tests/test_sync_core_manifest.py @@ -91,8 +91,7 @@ def test_removed_prompt_remains_expected_and_requires_tombstone(tmp_path) -> Non def test_protected_tombstone_accounts_for_removed_unit(tmp_path) -> None: root = _repository(tmp_path) - base = _commit(root, "base") - (root / "prompts/widget_python.prompt").unlink() + _commit(root, "base") tombstone = root / ".pdd/sync-tombstones.json" tombstone.write_text( json.dumps( @@ -110,6 +109,8 @@ def test_protected_tombstone_accounts_for_removed_unit(tmp_path) -> None: ] ) ) + base = _commit(root, "authorize decommission") + (root / "prompts/widget_python.prompt").unlink() head = _commit(root, "decommission widget") manifest = build_unit_manifest(root, base_ref=base, head_ref=head) assert not any("lacks a complete" in reason for reason in manifest.invalid_reasons) @@ -119,8 +120,7 @@ def test_protected_tombstone_accounts_for_removed_unit(tmp_path) -> None: def test_incomplete_tombstone_cannot_reduce_expected_managed(tmp_path) -> None: root = _repository(tmp_path) - base = _commit(root, "base") - (root / "prompts/widget_python.prompt").unlink() + _commit(root, "base") (root / ".pdd/sync-tombstones.json").write_text( json.dumps( [ @@ -134,12 +134,62 @@ def test_incomplete_tombstone_cannot_reduce_expected_managed(tmp_path) -> None: ] ) ) + base = _commit(root, "attempt decommission authorization") + (root / "prompts/widget_python.prompt").unlink() head = _commit(root, "incomplete tombstone") manifest = build_unit_manifest(root, base_ref=base, head_ref=head) assert len(manifest.expected_managed) == 1 assert any("complete IN_SYNC tombstone" in item for item in manifest.invalid_reasons) +def test_protected_registry_preserves_and_authorizes_denominator_transition( + tmp_path, +) -> None: + root = _repository(tmp_path) + _commit(root, "base") + (root / ".pdd/expected-managed.json").write_text( + json.dumps( + { + "schema_version": 1, + "units": [ + { + "prompt_path": "prompts/widget_python.prompt", + "language_id": "python", + } + ], + } + ) + ) + (root / ".pdd/sync-tombstones.json").write_text( + json.dumps( + [ + { + "prompt_path": "prompts/widget_python.prompt", + "artifact_paths": [ + "prompts/widget_python.prompt", + "src/widget.py", + ], + "rationale": "Widget was deliberately retired", + "owner": "sync-owner@example.com", + "baseline_status": "IN_SYNC", + } + ] + ) + ) + authorized = _commit(root, "protect decommission authorization") + (root / "prompts/widget_python.prompt").unlink() + (root / "src/widget.py").unlink() + (root / "architecture.json").write_text("[]\n") + removed = _commit(root, "remove authorized widget") + + transition = build_unit_manifest(root, base_ref=authorized, head_ref=removed) + stable = build_unit_manifest(root, base_ref=removed, head_ref=removed) + assert transition.expected_managed == () + assert stable.expected_managed == () + assert transition.invalid_reasons == () + assert stable.invalid_reasons == () + + def test_architecture_mapping_prefers_project_prompt_root_over_duplicate_leaf( tmp_path, ) -> None: diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index 2e4672853..a31811f33 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -98,6 +98,9 @@ def _repository(tmp_path: Path) -> tuple[Path, str]: } ) ) + (root / ".pdd/sync-policy.json").write_text( + json.dumps({"schema_version": 1, "enforcement": "active"}) + ) (root / ".pdd/verification-profiles.json").write_text( json.dumps( { @@ -111,7 +114,7 @@ def _repository(tmp_path: Path) -> tuple[Path, str]: "obligation_id": "pytest", "kind": "test", "validator_id": "pytest", - "validator_config_digest": "pytest-v1", + "validator_config_digest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requirement_ids": ["REQ-1"], "artifact_paths": ["tests/test_widget.py"], } @@ -321,6 +324,18 @@ def test_nested_config_cannot_bypass_repository_canonical_finalizer( ) +def test_repository_identity_does_not_activate_enforcement_without_policy( + tmp_path, +) -> None: + root, _commit = _repository(tmp_path) + policy = root / ".pdd/sync-policy.json" + policy.unlink() + _git(root, "add", "-u", ".pdd/sync-policy.json") + _git(root, "commit", "-q", "-m", "remove activation policy") + + assert canonical_sync_enabled(root) is False + + def test_code_drift_cannot_reuse_old_attestation(tmp_path) -> None: root, commit = _repository(tmp_path) _finalize_trusted_baseline(root, commit) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index fda8d6ead..3e9420644 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -15,6 +15,7 @@ VerificationProfile, run_profile, ) +from pdd.sync_core.runner import pytest_validator_config_digest UNIT = UnitId("repository-1", PurePosixPath("prompts/widget_python.prompt"), "python") @@ -39,14 +40,15 @@ def _repository(tmp_path: Path, test_content: str) -> tuple[Path, str]: return root, _git(root, "rev-parse", "HEAD") -def _profile() -> VerificationProfile: +def _profile(root: Path, ref: str) -> VerificationProfile: + paths = (PurePosixPath("tests/test_widget.py"),) obligation = VerificationObligation( "pytest", "test", "pytest", - "pytest-v1", + pytest_validator_config_digest(root, ref, paths), ("REQ-1",), - (PurePosixPath("tests/test_widget.py"),), + paths, ) return VerificationProfile(UNIT, (obligation,), ("REQ-1",), "profile-v1") @@ -54,7 +56,7 @@ def _profile() -> VerificationProfile: def _run(root: Path, base: str, head: str): return run_profile( root, - _profile(), + _profile(root, base), RunBinding("snapshot-v1", base, head), AttestationIssue( AttestationSigner("trusted-ci", b"v" * 32), @@ -114,8 +116,8 @@ def test_candidate_modified_conftest_cannot_self_certify(tmp_path) -> None: _git(root, "commit", "-q", "-m", "candidate fixture") head = _git(root, "rev-parse", "HEAD") _envelope, executions = _run(root, base, head) - assert executions[0].outcome is EvidenceOutcome.QUARANTINED - assert "tests/conftest.py" in executions[0].detail + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "config digest" in executions[0].detail def test_untracked_conftest_cannot_influence_checked_sha(tmp_path) -> None: diff --git a/tests/test_sync_core_snapshot.py b/tests/test_sync_core_snapshot.py index 67c6791d4..fcd66b490 100644 --- a/tests/test_sync_core_snapshot.py +++ b/tests/test_sync_core_snapshot.py @@ -41,6 +41,21 @@ def _repository(tmp_path: Path, *, query: bool = False) -> tuple[Path, str]: (root / "src/widget.py").write_text("value = 1\n") (root / "tests/test_widget.py").write_text("def test_widget(): pass\n") (root / "tests/test_widget_e2e.py").write_text("def test_e2e(): pass\n") + (root / "notes.md").write_text("Human-owned release notes\n") + (root / ".pdd/sync-ownership.json").write_text( + json.dumps( + { + "rules": [ + { + "pattern": "notes.md", + "inventory": "HUMAN_OWNED", + "role": "documentation", + "owner": "docs@example.com", + } + ] + } + ) + ) os.chmod(root / "tests/test_widget_e2e.py", 0o755) (root / "architecture.json").write_text( json.dumps( @@ -60,7 +75,7 @@ def _repository(tmp_path: Path, *, query: bool = False) -> tuple[Path, str]: "obligation_id": "tests", "kind": "test", "validator_id": "pytest", - "validator_config_digest": "pytest-v1", + "validator_config_digest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requirement_ids": ["REQ-1"], "artifact_paths": [ "tests/test_widget.py", @@ -103,3 +118,26 @@ def test_query_expansion_cannot_receive_trusted_verified_status(tmp_path) -> Non profile = load_verification_profiles(root, manifest).profiles[0] snapshot = build_unit_snapshot(root, manifest, manifest.managed_units[0], profile) assert snapshot.nondeterministic_inputs is True + + +def test_unrelated_human_owned_change_does_not_invalidate_unit_snapshot(tmp_path) -> None: + root, base = _repository(tmp_path) + first_manifest = build_unit_manifest(root, base_ref=base, head_ref=base) + first_profile = load_verification_profiles(root, first_manifest).profiles[0] + first = build_unit_snapshot( + root, first_manifest, first_manifest.managed_units[0], first_profile + ) + + (root / "notes.md").write_text("Unrelated human-owned update\n") + _git(root, "add", "notes.md") + _git(root, "commit", "-q", "-m", "update release notes") + head = _git(root, "rev-parse", "HEAD") + second_manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + second_profile = load_verification_profiles(root, second_manifest).profiles[0] + second = build_unit_snapshot( + root, second_manifest, second_manifest.managed_units[0], second_profile + ) + + assert first_manifest.digest() != second_manifest.digest() + assert first.manifest_digest == second.manifest_digest + assert first.digest() == second.digest() diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index 031596686..004691ac1 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -123,6 +123,23 @@ def test_later_corrupt_blob_is_detected_before_first_install(tmp_path) -> None: assert not (tmp_path / ".pdd/evidence/widget.json").exists() +def test_later_corrupt_rollback_blob_is_detected_before_first_install(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + evidence = tmp_path / ".pdd/evidence/widget.json" + evidence.parent.mkdir(parents=True) + evidence.write_text("old evidence\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-rollback-corrupt", _writes()) + transaction = tmp_path / ".pdd/transactions/tx-rollback-corrupt" + (transaction / "rollback-1.blob").write_bytes(b"attacker bytes\n") + with pytest.raises(TransactionError, match="rollback transaction blob is corrupt"): + manager.commit("tx-rollback-corrupt") + assert target.read_text() == "value = 1\n" + assert evidence.read_text() == "old evidence\n" + + def test_recovery_rolls_back_partial_install_when_later_blob_is_corrupt(tmp_path) -> None: (tmp_path / "src").mkdir() target = tmp_path / "src/widget.py" diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index f83140a71..a24b0df6a 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -1,5 +1,8 @@ """Adversarial tests for trusted semantic evidence issuance.""" +import base64 +import json +import subprocess from dataclasses import replace from datetime import datetime, timedelta, timezone from pathlib import PurePosixPath @@ -14,9 +17,11 @@ AttestationTrustPolicy, EvidenceOutcome, FileReplayStore, + RemoteAttestationSigner, UnitId, ) from pdd.sync_core.trust import ValidationEvidence +from pdd.sync_core.finalize import attestation_signer_from_environment from pdd.sync_core.types import ObligationEvidence @@ -65,6 +70,55 @@ def test_valid_attestation_produces_sealed_evidence() -> None: assert evidence.attestation_id == "attestation-1" +def test_remote_attestation_authority_signature_is_verified(monkeypatch) -> None: + request = AttestationRequest( + "remote-attestation", + _binding(), + (ObligationEvidence("test", EvidenceOutcome.PASS),), + "remote-nonce", + NOW, + ) + + def remote_sign(command, *, input, capture_output, check): + assert command == ("protected-attestation-sign",) + assert capture_output is True and check is False + return subprocess.CompletedProcess( + command, + 0, + stdout=SIGNER.sign_bytes(input).encode("ascii"), + stderr=b"", + ) + + monkeypatch.setattr("pdd.sync_core.trust.subprocess.run", remote_sign) + signer = RemoteAttestationSigner( + SIGNER.issuer, SIGNER.public_key_bytes(), ("protected-attestation-sign",) + ) + envelope = signer.issue(request) + evidence = AttestationTrustPolicy( + {SIGNER.issuer: SIGNER.public_key_bytes()} + ).verify(envelope, request.binding, now=NOW) + assert evidence.attestation_id == request.attestation_id + + +def test_attestation_environment_forbids_local_private_key(monkeypatch) -> None: + monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-secret") + with pytest.raises(ValueError, match="local attestation signing keys are forbidden"): + attestation_signer_from_environment() + + +def test_attestation_environment_builds_remote_authority(monkeypatch) -> None: + monkeypatch.setenv("PDD_ATTESTATION_ISSUER", SIGNER.issuer) + monkeypatch.setenv( + "PDD_ATTESTATION_PUBLIC_KEY", + base64.b64encode(SIGNER.public_key_bytes()).decode("ascii"), + ) + monkeypatch.setenv( + "PDD_ATTESTATION_SIGNER_COMMAND", + json.dumps(["protected-attestation-sign"]), + ) + assert isinstance(attestation_signer_from_environment(), RemoteAttestationSigner) + + def test_evidence_cannot_be_caller_asserted() -> None: with pytest.raises(TypeError, match="AttestationTrustPolicy"): ValidationEvidence( @@ -91,7 +145,7 @@ def test_expired_attestation_is_rejected() -> None: def test_overlong_attestation_lifetime_is_rejected() -> None: - envelope = _envelope(nonce="nonce-long", lifetime=timedelta(hours=2)) + envelope = _envelope(nonce="nonce-long", lifetime=timedelta(days=9)) with pytest.raises(AttestationError, match="exceeds policy"): _verify(AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}), envelope) @@ -125,12 +179,12 @@ def test_nonce_reuse_by_different_attestation_is_rejected() -> None: _verify(policy, second) -def test_exact_signed_statement_replay_is_rejected() -> None: +def test_exact_signed_statement_recheck_is_idempotent() -> None: policy = AttestationTrustPolicy({"trusted-ci": PUBLIC_KEY}) envelope = _envelope() - _verify(policy, envelope) - with pytest.raises(AttestationError, match="replayed"): - _verify(policy, envelope) + first = _verify(policy, envelope) + second = _verify(policy, envelope) + assert first.attestation_id == second.attestation_id == "attestation-1" def test_durable_nonce_collision_is_rejected_across_policy_instances(tmp_path) -> None: @@ -157,6 +211,19 @@ def test_durable_nonce_collision_is_rejected_across_policy_instances(tmp_path) - assert path.stat().st_mode & 0o777 == 0o600 +def test_durable_exact_statement_recheck_is_idempotent(tmp_path) -> None: + path = tmp_path / "external/replay.json" + envelope = _envelope() + first = AttestationTrustPolicy( + {"trusted-ci": PUBLIC_KEY}, replay_store=FileReplayStore(path) + ) + second = AttestationTrustPolicy( + {"trusted-ci": PUBLIC_KEY}, replay_store=FileReplayStore(path) + ) + _verify(first, envelope) + assert _verify(second, envelope).attestation_id == envelope.attestation_id + + @pytest.mark.parametrize( "override", [ diff --git a/tests/test_sync_core_verification_profiles.py b/tests/test_sync_core_verification_profiles.py index 2e5119cd5..4ddc44a2f 100644 --- a/tests/test_sync_core_verification_profiles.py +++ b/tests/test_sync_core_verification_profiles.py @@ -145,7 +145,7 @@ def test_profile_cannot_invent_smaller_requirement_universe(tmp_path) -> None: assert profiles.coverage == 0.0 -def test_prompt_without_explicit_ids_uses_full_contract_digest(tmp_path) -> None: +def test_prompt_without_explicit_ids_requires_human_attestation(tmp_path) -> None: root = _repository(tmp_path) prompt = root / "prompts/widget_python.prompt" prompt.write_text("Build a widget with validated input.\n") @@ -157,5 +157,15 @@ def test_prompt_without_explicit_ids_uses_full_contract_digest(tmp_path) -> None (root / ".pdd/verification-profiles.json").write_text(json.dumps(profile)) commit = _commit(root, "contract digest") profiles = load_verification_profiles(root, _manifest(root, commit, commit)) - assert profiles.invalid_reasons == () - assert profiles.coverage == 1.0 + assert any("profile is incomplete" in item for item in profiles.invalid_reasons) + assert profiles.coverage == 0.0 + + +def test_candidate_only_profile_cannot_approve_itself(tmp_path) -> None: + root = _repository(tmp_path) + base = _commit(root, "unprofiled base") + (root / ".pdd/verification-profiles.json").write_text(json.dumps(_profile())) + head = _commit(root, "candidate profile") + profiles = load_verification_profiles(root, _manifest(root, base, head)) + assert profiles.coverage == 0.0 + assert any("lacks protected approval" in item for item in profiles.invalid_reasons) From ece9252ed7cc155c73490f6726debe555478c05c Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 16:57:34 -0700 Subject: [PATCH 003/128] fix: harden sync transaction destination writes --- pdd/sync_core/transaction.py | 194 ++++++++++++++++++++-------- tests/test_sync_core_transaction.py | 35 +++++ 2 files changed, 177 insertions(+), 52 deletions(-) diff --git a/pdd/sync_core/transaction.py b/pdd/sync_core/transaction.py index 3a986893f..d5a8c7659 100644 --- a/pdd/sync_core/transaction.py +++ b/pdd/sync_core/transaction.py @@ -8,7 +8,8 @@ import shutil import stat import tempfile -from contextlib import ExitStack +import uuid +from contextlib import ExitStack, contextmanager from dataclasses import dataclass from enum import Enum from pathlib import Path, PurePosixPath @@ -85,16 +86,27 @@ def _git_mode(mode: int) -> str: return "100755" if mode & executable else "100644" -def _file_state(path: Path) -> FileState: - if not path.exists() and not path.is_symlink(): +def _descriptor_file_state(parent_fd: int, name: str) -> FileState: + """Read one destination without following its final path component.""" + try: + mode = os.stat(name, dir_fd=parent_fd, follow_symlinks=False).st_mode + except FileNotFoundError: return FileState(False, None, None, "missing") - mode = path.lstat().st_mode if stat.S_ISLNK(mode): return FileState(True, None, "120000", "symlink") if not stat.S_ISREG(mode): return FileState(True, None, None, "special") - content = path.read_bytes() - return FileState(True, _digest(content), _git_mode(mode), "regular") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(name, flags, dir_fd=parent_fd) + try: + opened_mode = os.fstat(descriptor).st_mode + if not stat.S_ISREG(opened_mode): + return FileState(True, None, None, "special") + with os.fdopen(descriptor, "rb", closefd=False) as handle: + content = handle.read() + return FileState(True, _digest(content), _git_mode(opened_mode), "regular") + finally: + os.close(descriptor) def _state_payload(state: FileState) -> dict[str, object]: @@ -149,6 +161,115 @@ def _transaction_dir(self, transaction_id: str) -> Path: raise TransactionError("transaction ID contains unsafe characters") return self.state_root / transaction_id + def _canonical_relpath(self, relpath: PurePosixPath) -> PurePosixPath: + resolved = self.policy.resolve(relpath, allow_missing=True) + try: + relative = resolved.canonical_path.relative_to(self.checkout_root) + except ValueError as exc: + raise TransactionError(f"destination escapes checkout: {relpath}") from exc + return PurePosixPath(relative.as_posix()) + + @contextmanager + def _parent_descriptor( + self, relpath: PurePosixPath, *, create: bool = False + ): + """Pin and no-follow every parent directory for one destination.""" + canonical = self._canonical_relpath(relpath) + root_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_flags = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + descriptors: list[int] = [] + try: + current = os.open(self.checkout_root, root_flags) + descriptors.append(current) + for component in canonical.parts[:-1]: + try: + child = os.open(component, directory_flags, dir_fd=current) + except FileNotFoundError: + if not create: + raise + os.mkdir(component, mode=0o755, dir_fd=current) + os.fsync(current) + child = os.open(component, directory_flags, dir_fd=current) + descriptors.append(child) + current = child + yield current, canonical.name + except OSError as exc: + raise TransactionConflict( + f"destination parent changed or is unsafe: {relpath}" + ) from exc + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + def _destination_state(self, relpath: PurePosixPath) -> FileState: + try: + with self._parent_descriptor(relpath) as (parent_fd, name): + return _descriptor_file_state(parent_fd, name) + except TransactionConflict as exc: + if isinstance(exc.__cause__, FileNotFoundError): + return FileState(False, None, None, "missing") + raise + + def _read_destination(self, relpath: PurePosixPath) -> bytes: + with self._parent_descriptor(relpath) as (parent_fd, name): + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(name, flags, dir_fd=parent_fd) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise TransactionError( + f"destination is not a regular file: {relpath}" + ) + with os.fdopen(descriptor, "rb", closefd=False) as handle: + return handle.read() + finally: + os.close(descriptor) + + def _replace_destination( + self, relpath: PurePosixPath, content: bytes, git_mode: str, suffix: str + ) -> None: + with self._parent_descriptor(relpath, create=True) as (parent_fd, name): + temporary_name = f".{name}.{uuid.uuid4().hex}.{suffix}" + flags = ( + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0) + ) + descriptor = os.open(temporary_name, flags, 0o600, dir_fd=parent_fd) + try: + os.fchmod(descriptor, 0o755 if git_mode == "100755" else 0o644) + with os.fdopen(descriptor, "wb", closefd=False) as handle: + handle.write(content) + handle.flush() + os.fsync(descriptor) + os.replace( + temporary_name, + name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + ) + os.fsync(parent_fd) + except BaseException: + try: + os.unlink(temporary_name, dir_fd=parent_fd) + except FileNotFoundError: + pass + raise + finally: + os.close(descriptor) + + def _unlink_destination(self, relpath: PurePosixPath) -> None: + with self._parent_descriptor(relpath) as (parent_fd, name): + try: + os.unlink(name, dir_fd=parent_fd) + except FileNotFoundError: + return + os.fsync(parent_fd) + @staticmethod def _journal_path(transaction_dir: Path) -> Path: return transaction_dir / "journal.json" @@ -215,8 +336,7 @@ def _entry( index: int, write: PlannedWrite, ) -> dict[str, object]: - resolved = self.policy.resolve(write.relpath, allow_missing=True) - before = _file_state(resolved.canonical_path) + before = self._destination_state(write.relpath) if write.expected is not None and before != write.expected: raise TransactionConflict( f"destination changed before prepare: {write.relpath}" @@ -227,7 +347,10 @@ def _entry( rollback_name = f"rollback-{index}.blob" if before.exists else None self._write_blob(transaction_dir / prepared_name, write.content) if rollback_name: - self._write_blob(transaction_dir / rollback_name, resolved.canonical_path.read_bytes()) + self._write_blob( + transaction_dir / rollback_name, + self._read_destination(write.relpath), + ) return { "relpath": write.relpath.as_posix(), "desired_digest": _digest(write.content), @@ -253,9 +376,7 @@ def prepare( for write in writes } current_states = { - write.relpath: _file_state( - self.policy.resolve(write.relpath, allow_missing=True).canonical_path - ) + write.relpath: self._destination_state(write.relpath) for write in writes } for write in writes: @@ -307,8 +428,7 @@ def _install_entry( entry: dict[str, object], ) -> None: relpath = PurePosixPath(str(entry["relpath"])) - resolved = self.policy.resolve(relpath, allow_missing=True) - current = _file_state(resolved.canonical_path) + current = self._destination_state(relpath) desired = FileState( True, str(entry["desired_digest"]), @@ -322,30 +442,13 @@ def _install_entry( raise TransactionError("transaction precondition is malformed") if current != _parse_state(precondition_payload): raise TransactionConflict(f"destination changed: {relpath}") - resolved.canonical_path.parent.mkdir(parents=True, exist_ok=True) prepared = transaction_dir / str(entry["prepared_blob"]) if prepared.is_symlink() or not prepared.is_file(): raise TransactionError(f"prepared transaction blob is unsafe: {relpath}") content = prepared.read_bytes() if _digest(content) != desired.digest: raise TransactionError(f"prepared transaction blob is corrupt: {relpath}") - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{resolved.canonical_path.name}.", - suffix=".pdd-tmp", - dir=resolved.canonical_path.parent, - ) - temporary = Path(temporary_name) - try: - os.fchmod(descriptor, 0o755 if desired.git_mode == "100755" else 0o644) - with os.fdopen(descriptor, "wb") as handle: - handle.write(content) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, resolved.canonical_path) - fsync_directory(resolved.canonical_path.parent) - except BaseException: - temporary.unlink(missing_ok=True) - raise + self._replace_destination(relpath, content, desired.git_mode, "pdd-tmp") def _validate_prepared_entries( self, transaction_dir: Path, entries: list[object] @@ -385,15 +488,13 @@ def _restore_entries( if not isinstance(item, dict) or item.get("installed") is not True: continue relpath = PurePosixPath(str(item["relpath"])) - destination = self.policy.resolve(relpath, allow_missing=True).canonical_path precondition = item.get("precondition") if not isinstance(precondition, dict): raise TransactionError("transaction rollback precondition is malformed") before = _parse_state(precondition) rollback_name = item.get("rollback_blob") if not before.exists: - destination.unlink(missing_ok=True) - fsync_directory(destination.parent) + self._unlink_destination(relpath) continue rollback = transaction_dir / str(rollback_name) if rollback.is_symlink() or not rollback.is_file(): @@ -401,21 +502,12 @@ def _restore_entries( content = rollback.read_bytes() if _digest(content) != before.digest: raise TransactionError(f"rollback transaction blob is corrupt: {relpath}") - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{destination.name}.", suffix=".pdd-rollback", dir=destination.parent + self._replace_destination( + relpath, + content, + str(before.git_mode), + "pdd-rollback", ) - temporary = Path(temporary_name) - try: - os.fchmod(descriptor, 0o755 if before.git_mode == "100755" else 0o644) - with os.fdopen(descriptor, "wb") as handle: - handle.write(content) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, destination) - fsync_directory(destination.parent) - except BaseException: - temporary.unlink(missing_ok=True) - raise def commit( self, @@ -441,9 +533,7 @@ def commit( if not isinstance(item, dict): raise TransactionError("transaction entry is malformed") relpath = PurePosixPath(str(item["relpath"])) - current = _file_state( - self.policy.resolve(relpath, allow_missing=True).canonical_path - ) + current = self._destination_state(relpath) precondition = item.get("precondition") if not isinstance(precondition, dict) or current != _parse_state(precondition): raise TransactionConflict(f"destination changed: {relpath}") diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index 004691ac1..6020d5677 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -196,3 +196,38 @@ def test_secret_labeled_write_refuses_unencrypted_rollback(tmp_path) -> None: ) with pytest.raises(TransactionError, match="requires configured encryption"): TransactionManager(tmp_path).prepare("tx-secret", (secret,)) + + +def test_descriptor_time_parent_symlink_swap_cannot_redirect_commit( + tmp_path, monkeypatch +) -> None: + source = tmp_path / "src" + source.mkdir() + target = source / "widget.py" + target.write_text("value = 1\n") + outside = tmp_path.parent / f"{tmp_path.name}-outside" + outside.mkdir() + outside_target = outside / "widget.py" + outside_target.write_text("outside = True\n") + + manager = TransactionManager(tmp_path) + manager.prepare("tx-parent-swap", _writes()) + original = manager._canonical_relpath # pylint: disable=protected-access + armed = True + + def swap_after_resolution(relpath): + nonlocal armed + canonical = original(relpath) + if armed and relpath == PurePosixPath("src/widget.py"): + armed = False + source.rename(tmp_path / "src-before-swap") + source.symlink_to(outside, target_is_directory=True) + return canonical + + monkeypatch.setattr(manager, "_canonical_relpath", swap_after_resolution) + with pytest.raises(TransactionConflict, match="parent changed or is unsafe"): + manager.commit("tx-parent-swap") + + assert outside_target.read_text() == "outside = True\n" + assert (tmp_path / "src-before-swap/widget.py").read_text() == "value = 1\n" + assert not (tmp_path / ".pdd/evidence/widget.json").exists() From 8f876a2be968027a361b068e79b5c0671e958cb9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 17:04:51 -0700 Subject: [PATCH 004/128] fix: block unstaged protected sync mutations --- pdd/sync_core/scenario_harness.py | 39 +++++++++++++++++++++++++++++++ pdd/sync_orchestration.py | 27 +++++++-------------- tests/test_sync_orchestration.py | 21 +++++++++++++++++ 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/pdd/sync_core/scenario_harness.py b/pdd/sync_core/scenario_harness.py index 6ce3d7b66..3070b6fc5 100644 --- a/pdd/sync_core/scenario_harness.py +++ b/pdd/sync_core/scenario_harness.py @@ -363,6 +363,44 @@ def commit(manager: TransactionManager, transaction_id: str) -> str: raise AssertionError(f"concurrent sync was not serialized: {outcomes}") +def _descriptor_swap_race(writes: tuple[PlannedWrite, ...]) -> None: + # pylint: disable=protected-access + with tempfile.TemporaryDirectory(prefix="pdd-checker-descriptor-") as directory: + root = Path(directory) + source = root / "src" + outside = root / "outside" + source.mkdir() + outside.mkdir() + (source / "widget.py").write_text("old\n") + outside_target = outside / "widget.py" + outside_target.write_text("outside\n") + manager = TransactionManager(root) + manager.prepare("descriptor-race", writes) + canonical_relpath = manager._canonical_relpath + armed = True + + def swap_after_resolution(relpath: PurePosixPath) -> PurePosixPath: + nonlocal armed + canonical = canonical_relpath(relpath) + if armed and relpath == PurePosixPath("src/widget.py"): + armed = False + source.rename(root / "src-before-swap") + source.symlink_to(outside, target_is_directory=True) + return canonical + + manager._canonical_relpath = ( # type: ignore[method-assign] + swap_after_resolution + ) + try: + manager.commit("descriptor-race") + except TransactionConflict: + pass + else: + raise AssertionError("descriptor-time parent swap was not rejected") + if outside_target.read_text() != "outside\n": + raise AssertionError("descriptor-time parent swap redirected a write") + + def _transaction_crash_race(_arguments: argparse.Namespace) -> ScenarioResult: writes = ( PlannedWrite(PurePosixPath("src/widget.py"), b"new\n", "100644"), @@ -372,6 +410,7 @@ def _transaction_crash_race(_arguments: argparse.Namespace) -> ScenarioResult: _recover_after_crashes(writes) _external_write_race(writes) _concurrent_sync_race(writes) + _descriptor_swap_race(writes) return ScenarioResult("transaction-crash-race-recovery", "PASS") diff --git a/pdd/sync_orchestration.py b/pdd/sync_orchestration.py index f0f66a3ef..5db06ecd9 100644 --- a/pdd/sync_orchestration.py +++ b/pdd/sync_orchestration.py @@ -2673,24 +2673,15 @@ def sync_worker_logic(): test_output_excerpt: Optional[str] = None operation_rollback = None - from .continuous_sync import canonical_sync_enabled, project_root - - operation_root = project_root(pdd_files.get("prompt") or Path.cwd()) - if canonical_sync_enabled(operation_root): - rollback_paths = [] - for value in pdd_files.values(): - if isinstance(value, Path): - rollback_paths.append(value) - elif isinstance(value, list): - rollback_paths.extend( - item for item in value if isinstance(item, Path) - ) - for shared in ( - operation_root / "architecture.json", - operation_root / "project_dependencies.csv", - ): - rollback_paths.append(shared) - operation_rollback = OperationFileRollback(rollback_paths) + from .continuous_sync import canonical_sync_enabled + + operation_start = pdd_files.get("prompt") or Path.cwd() + if canonical_sync_enabled(Path(operation_start)): + raise RuntimeError( + "protected canonical sync blocks legacy production mutation; " + "use read-only reporting or trusted finalization until the " + "staged repair executor is enabled" + ) # Issue #159 fix: Use atomic state for consistent run_report + fingerprint writes set_current_operation(operation) diff --git a/tests/test_sync_orchestration.py b/tests/test_sync_orchestration.py index 16d9c8664..1b841fec0 100644 --- a/tests/test_sync_orchestration.py +++ b/tests/test_sync_orchestration.py @@ -3098,6 +3098,27 @@ def capture_append(_b, _l, entry, **_kwargs): # --- Coverage Target Selection Regression Tests --- + +def test_protected_canonical_mode_blocks_legacy_generator_before_write( + orchestration_fixture, +): + """Protected repositories must not run generators against production paths.""" + orchestration_fixture["sync_determine_operation"].side_effect = [ + SyncDecision(operation="generate", reason="prompt changed") + ] + with patch("pdd.continuous_sync.canonical_sync_enabled", return_value=True): + result = sync_orchestration( + basename="calculator", + language="python", + quiet=True, + budget=1.0, + ) + + assert result["success"] is False + assert "blocks legacy production mutation" in " ".join(result["errors"]) + orchestration_fixture["code_generator_main"].assert_not_called() + + class TestCoverageTargetSelection: """Regression tests for selecting the correct `--cov` target.""" From 3abece70183d6978d117d4560ffe5e8cba786e1d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 17:09:08 -0700 Subject: [PATCH 005/128] feat: bind candidate wheel to sync certificate --- pdd/sync_core/certificate.py | 22 ++++++++++- pdd/sync_core/lifecycle.py | 8 +++- tests/test_sync_core_certificate.py | 59 ++++++++++++++++++++++++++--- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index d0eabaa54..f6ffdceec 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -76,6 +76,7 @@ class RepositoryTarget: @dataclass(frozen=True) class LifecycleResult: + # pylint: disable=too-many-instance-attributes """Externally produced required-scenario and test-outcome totals.""" failed: int @@ -85,6 +86,7 @@ class LifecycleResult: post_repair_second_run_writes: int post_merge_tree_changes: int missing_scenarios: tuple[str, ...] = () + candidate_wheel_sha256: str = "" @dataclass(frozen=True) @@ -393,6 +395,7 @@ def _complete_nightly( "required_tests_skipped_or_xfailed", "collection_errors", "timeouts", + "candidate_wheel_sha256", } return ( required_counts <= counts.keys() @@ -519,6 +522,11 @@ def _scan_predicate( and extra["pdd_cloud_vendored_sync_semantics"] == 0 and lifecycle.post_repair_second_run_writes == 0 and lifecycle.post_merge_tree_changes == 0 + and len(lifecycle.candidate_wheel_sha256) == 64 + and all( + character in "0123456789abcdef" + for character in lifecycle.candidate_wheel_sha256 + ) and extra["nightly_observation_complete"] == 1 ) @@ -590,7 +598,7 @@ def _canonical_report_predicate(report: dict[str, Any]) -> bool: def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool]: - # pylint: disable=too-many-return-statements + # pylint: disable=too-many-locals,too-many-return-statements if body.get("schema_version") != 2: return False, False checker = body.get("checker") @@ -645,9 +653,20 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] for name in lifecycle_names ) or not isinstance(lifecycle_payload.get("missing_scenarios"), list): return False, False + candidate_wheel_sha256 = lifecycle_payload.get("candidate_wheel_sha256") + if ( + not isinstance(candidate_wheel_sha256, str) + or len(candidate_wheel_sha256) != 64 + or any( + character not in "0123456789abcdef" + for character in candidate_wheel_sha256 + ) + ): + return False, False lifecycle = LifecycleResult( *(lifecycle_payload[name] for name in lifecycle_names), tuple(lifecycle_payload["missing_scenarios"]), + candidate_wheel_sha256, ) extra_names = { "pdd_cloud_vendored_sync_semantics", @@ -771,6 +790,7 @@ def _build_global_certificate_from_targets( "post_repair_second_run_writes": lifecycle.post_repair_second_run_writes, "post_merge_tree_changes": lifecycle.post_merge_tree_changes, "missing_scenarios": list(lifecycle.missing_scenarios), + "candidate_wheel_sha256": lifecycle.candidate_wheel_sha256, }, } body["scan_ok"] = all(report.get("ok") for report in reports) and _scan_predicate( diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 37e2c129c..f323c0df7 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import os import subprocess @@ -39,6 +40,7 @@ def _failed_result(*, timeout: bool = False) -> LifecycleResult: 1, 1, tuple(sorted(REQUIRED_SCENARIOS)), + "", ) @@ -81,7 +83,7 @@ def _install_candidate_wheel(temporary: Path, home: Path, wheel: Path) -> Path | environment = temporary / "candidate-venv" isolated = _isolated_lifecycle_environment(home) created = subprocess.run( - [sys.executable, "-m", "venv", "--system-site-packages", str(environment)], + [sys.executable, "-m", "venv", str(environment)], capture_output=True, text=True, check=False, @@ -98,7 +100,8 @@ def _install_candidate_wheel(temporary: Path, home: Path, wheel: Path) -> Path | "-m", "pip", "install", - "--no-deps", + "--only-binary=:all:", + "--disable-pip-version-check", "--force-reinstall", str(wheel.resolve()), ], @@ -194,4 +197,5 @@ def run_lifecycle_matrix( ), sum(int(row["post_merge_tree_changes"]) for row in results.values()), missing, + hashlib.sha256(Path(candidate_wheel).read_bytes()).hexdigest(), ) diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 512767f3a..7f2044b03 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -29,6 +29,7 @@ "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v-test-checker", ) OBSERVATION = NightlyObservation(True, True, 0, True, "BLOCKED", 0) +CANDIDATE_WHEEL_SHA256 = "d" * 64 @pytest.fixture(autouse=True) @@ -71,7 +72,7 @@ def _report(name): def _nightly(path, signer, count=7, *, include_observation=True): - start = datetime(2026, 7, 3, tzinfo=timezone.utc) + start = datetime.now(timezone.utc) - timedelta(days=count - 1) rows = [] for index in range(count): reports = [ @@ -108,6 +109,7 @@ def _nightly(path, signer, count=7, *, include_observation=True): "post_repair_second_run_writes": 0, "post_merge_tree_changes": 0, "missing_scenarios": [], + "candidate_wheel_sha256": CANDIDATE_WHEEL_SHA256, } body = { "schema_version": 2, @@ -208,7 +210,9 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc monkeypatch.setattr( "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 ) - lifecycle = LifecycleResult(0, 0, 0, 0, 0, 0) + lifecycle = LifecycleResult( + 0, 0, 0, 0, 0, 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256 + ) certificate = build_global_certificate( ( RepositoryTarget("pdd", pdd), @@ -239,6 +243,23 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, ) is True + assert ( + certificate["lifecycle"]["candidate_wheel_sha256"] + == CANDIDATE_WHEEL_SHA256 + ) + certificate["lifecycle"]["candidate_wheel_sha256"] = "not-a-wheel-digest" + body = {key: value for key, value in certificate.items() if key != "signature"} + certificate["signature"]["value"] = signer.sign_bytes( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ) + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + ) is False certificate["counts"]["managed_units"] = 0 assert verify_global_certificate( certificate, @@ -266,7 +287,9 @@ def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> monkeypatch.setattr( "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 ) - lifecycle = LifecycleResult(0, 0, 0, 0, 0, 0, ("merge-group",)) + lifecycle = LifecycleResult( + 0, 0, 0, 0, 0, 0, ("merge-group",), CANDIDATE_WHEEL_SHA256 + ) certificate = build_global_certificate( ( RepositoryTarget("pdd", tmp_path / "pdd"), @@ -322,7 +345,15 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( ), GlobalCertificateOptions( tmp_path / "replay", - LifecycleResult(0, 0, 0, 0, 0, 0), + LifecycleResult( + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + ), nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, @@ -467,7 +498,15 @@ def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( ), GlobalCertificateOptions( tmp_path / "replay", - LifecycleResult(0, 0, 0, 0, 0, 0), + LifecycleResult( + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + ), nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, @@ -504,7 +543,15 @@ def test_signed_nightly_rows_without_observations_do_not_extend_streak( ), GlobalCertificateOptions( tmp_path / "replay", - LifecycleResult(0, 0, 0, 0, 0, 0), + LifecycleResult( + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + ), nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, From a6e2eaf4da4358fef458b9105d3f49ef6064b9d2 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 17:17:00 -0700 Subject: [PATCH 006/128] docs: record round nine sync blockers --- docs/global_sync_resolution_plan.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/global_sync_resolution_plan.md b/docs/global_sync_resolution_plan.md index 86ccceb26..9b0a7c6af 100644 --- a/docs/global_sync_resolution_plan.md +++ b/docs/global_sync_resolution_plan.md @@ -73,12 +73,13 @@ of Done or authorize a partial green certificate. | Exact certification | Independent exact-SHA clones, clean-tree/ancestry checks, strict fresh green certificate verification, exact issuer/repository refs, and remote-signature verification without local private keys | implemented and unit tested; protected signer service absent | | Trusted test runner | Strict pytest invocation; ambient options/plugins disabled; config, `conftest.py`, and transitive local helper closure signed; protected and candidate node IDs compared before each protected node executes independently; candidate-only profiles rejected and actual pytest config digest checked | implemented for pytest only; non-pytest and threshold-human adapters remain | | Temporal proof | Signed rows require complete predicates, repository identity, SHA ancestry, consecutive UTC dates, deleted-ledger full scans, normal no-op observations, injected canary outcomes, and zero-write reruns; exact immutable attestation rechecks are idempotent while nonce rebinding fails | implemented locally; protected timestamped storage and seven real nights absent | -| Lifecycle harness | A released checker installs the exact candidate wheel in a separate credential-free environment, drives public candidate report/recovery commands, and independently runs the required scenario registry and real cloud canary | split-install wheel run: one expected cloud-canary failure, zero skips/errors/timeouts/no-op writes/tree changes/missing scenarios; protected release/workflow deployment remains | -| PDD verification | `290 passed` for `tests/test_sync_core_*.py` with no skips; broad compatibility suite reached 1,145 pass/4 coverage-capture failures and all four focused regressions pass after the activation fast-path repair; unit snapshots no longer drift on unrelated human-owned changes | component green, E2E red | +| Lifecycle harness | A released checker installs the exact candidate wheel in a separate credential-free environment, binds its SHA-256 into the signed predicate, drives public candidate report/recovery commands, and independently runs the required scenario registry and real cloud canary | split-install wheel run: one expected cloud-canary failure; clean candidate-venv diagnostic adds only the expected source-checker provenance failure; zero skips/errors/timeouts/no-op writes/tree changes/missing scenarios; protected release/workflow deployment remains | +| Protected mutation | Transaction commits use pinned parent descriptors, no-follow traversal, descriptor-relative replace/unlink, full blob preflight, and a released-harness symlink-swap injection; legacy generators are blocked before invocation when protected canonical policy is active | safe fail-closed path implemented; staged repair executor remains absent | +| PDD verification | `291 passed` for `tests/test_sync_core_*.py` with no skips; broad compatibility suite reached 1,145 pass/4 coverage-capture failures and all four focused regressions pass after the activation fast-path repair; unit snapshots no longer drift on unrelated human-owned changes | component green, E2E red | | pdd_cloud boundary | Exact-tree path/Python/prompt/architecture scan reports zero forbidden vendored semantics at `677a9c88f`; the legacy finalizer declarations and implementation are removed and current consumer installs are exact-version pinned | implemented; final pin must target the protected reviewed release | | pdd_cloud inventory | Protected registry/tombstone commit `39b78e487` retains 573 historical identities and authorizes only the retired finalizer; both transition and stable manifests report 572 managed = 572 expected, zero invalid, and zero unaccounted | profiles/evidence migration remains | | Global certificate | Scan remains red for protected checker deployment, transactional staging, profile/evidence debt, and nightly history | correctly blocked | -| Adversarial review | xhigh round 8 returned `NOT APPROVED`; unit-global manifest coupling, candidate-wheel coverage, profile authority, replay/nightly observations, protected denominator, rollback preflight, and local signing-key findings have been addressed locally | generation staging/no-follow, release provenance, semantic ownership audit, adapter matrix, rollout, and seven nights remain | +| Adversarial review | xhigh round 9 returned `NOT APPROVED`; it confirmed exact-SHA candidate build attestation, protected release/nightly workflow, protected alias wiring, adapter completion, profile/evidence rollout, and seven real nights remain | no-follow commit and fail-closed unstaged mutation are implemented; release and rollout blockers remain | No acceptance claim is valid until the signed certificate recomputes `ok: true` for exact protected PDD and pdd_cloud SHAs and a verifier supplied with the From 1d7e3293387c37427bdbfbf34dcf6a8a00b5e43b Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 18:24:22 -0700 Subject: [PATCH 007/128] test(sync): reproduce foundation CI regressions --- tests/regression_public.sh | 7 +++++++ tests/test_ci_drift_heal.py | 21 +++++++++++++++++++++ tests/test_preprocess.py | 15 +++++++++++++++ tests/test_sync_core_includes.py | 7 +++++++ 4 files changed, 50 insertions(+) diff --git a/tests/regression_public.sh b/tests/regression_public.sh index f7c7e6e12..50b13b523 100755 --- a/tests/regression_public.sh +++ b/tests/regression_public.sh @@ -168,6 +168,13 @@ check_file_contains complex_preprocessed.prompt "Included deterministic content. check_file_contains complex_preprocessed.prompt "shell-output" "preprocess shell" check_file_not_contains complex_preprocessed.prompt "This internal comment should be stripped." "preprocess pdd comment stripping" +cat > bundled_docs_python.prompt <<'EOF' +docs/prompting_guide.md +EOF + +run_pdd preprocess --output bundled_docs_preprocessed.prompt bundled_docs_python.prompt +check_file_contains bundled_docs_preprocessed.prompt "PDD Mental Model" "bundled docs include" + log "4. Trace validation without model calls" cp "$REPO_ROOT/tests/fixtures/simple_math.py" simple_math.py run_pdd trace --output invalid_trace.log "$REPO_ROOT/tests/fixtures/simple_math_python.prompt" simple_math.py 99999 diff --git a/tests/test_ci_drift_heal.py b/tests/test_ci_drift_heal.py index d35e86f96..ac264040a 100644 --- a/tests/test_ci_drift_heal.py +++ b/tests/test_ci_drift_heal.py @@ -4377,6 +4377,27 @@ def test_dry_run_json_returns_one_when_not_ok(self, capsys): result = main(dry_run=True, as_json=True) assert result == 1 + def test_dry_run_json_redacts_secret_bearing_failure_data(self, capsys): + """The machine-readable dry-run report must not emit embedded secrets.""" + token = "ghp_" + "a" * 36 + report = { + "ok": False, + "summary": { + "metadata_stale": 0, + "conflicts": 0, + "unbaselined": 0, + "failures": 1, + }, + "failures": [f"remote command failed: Authorization: Bearer {token}"], + } + with patch("pdd.continuous_sync.build_report", return_value=report): + result = main(dry_run=True, as_json=True) + + assert result == 1 + output = capsys.readouterr().out + assert token not in output + assert "[REDACTED]" in output + class TestHealModuleConflict: def test_conflict_operation_returns_false(self): diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py index 1f873ee07..6693a01e9 100644 --- a/tests/test_preprocess.py +++ b/tests/test_preprocess.py @@ -775,6 +775,21 @@ def test_get_file_path() -> None: with pytest.raises(ValueError, match="absolute include path"): get_file_path(abs_path) + +def test_preprocess_resolves_bundled_prompting_guide_outside_checkout( + tmp_path, monkeypatch +) -> None: + """Bundled docs remain available when preprocessing from an empty project.""" + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("PDD_PATH", raising=False) + + processed = preprocess( + "docs/prompting_guide.md", + double_curly_brackets=False, + ) + + assert "PDD Mental Model" in processed + # Test for nested XML tags def test_nested_xml_tags() -> None: """Test handling of nested XML tags.""" diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py index 9f5183f53..9e6fd8913 100644 --- a/tests/test_sync_core_includes.py +++ b/tests/test_sync_core_includes.py @@ -61,6 +61,13 @@ def test_literal_include_tag_in_prose_does_not_consume_later_markup() -> None: assert [item.path for item in references] == ["docs/actual.md"] +@pytest.mark.timeout(1) +def test_malformed_include_text_is_bounded() -> None: + """Unterminated include markup cannot trigger superlinear parser backtracking.""" + text = " None: prompt = tmp_path / "widget_python.prompt" prompt.write_text(PROMPT, encoding="utf-8") From b7027e79040343182e6716a85a5b0343449d227e Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 18:28:34 -0700 Subject: [PATCH 008/128] fix(sync): resolve foundation CI regressions --- pdd/ci_drift_heal.py | 6 +- pdd/preprocess.py | 8 +-- pdd/sync_core/includes.py | 137 +++++++++++++++++++++++++++++--------- 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/pdd/ci_drift_heal.py b/pdd/ci_drift_heal.py index af8b675f0..1b80a8fbb 100644 --- a/pdd/ci_drift_heal.py +++ b/pdd/ci_drift_heal.py @@ -25,6 +25,8 @@ from rich.console import Console from rich.table import Table +from pdd.context_snapshot import redact_snapshot_text + console = Console() _HEAL_SUBPROCESS_TIMEOUT_DEFAULT = 2400 @@ -1890,11 +1892,11 @@ def main( return 0 if dry_run: from pdd.continuous_sync import build_report - import json as _json report = build_report(consumer="ci-heal", modules=modules) if as_json: - print(_json.dumps(report, indent=2, sort_keys=True)) + serialized = json.dumps(report, indent=2, sort_keys=True) + print(redact_snapshot_text(serialized)[0]) else: summary = report["summary"] console.print( diff --git a/pdd/preprocess.py b/pdd/preprocess.py index 7453a7a1d..e93e985ae 100644 --- a/pdd/preprocess.py +++ b/pdd/preprocess.py @@ -14,7 +14,6 @@ from .firecrawl_cache import get_firecrawl_cache from pdd.path_resolution import get_default_resolver from pdd.sync_core.includes import include_paths -from pdd.sync_core.path_policy import PathPolicy install() console = Console() @@ -432,15 +431,12 @@ def preprocess( def get_file_path(file_name: str) -> str: resolver = get_default_resolver() - project_root = resolver.resolve_project_root() relpath = Path(file_name) if relpath.is_absolute(): raise ValueError(f"absolute include path is not allowed: {file_name}") normalized = PurePosixPath(os.path.normpath(file_name).replace(os.sep, "/")) - resolved = PathPolicy(project_root).resolve( - normalized, allow_missing=True - ).canonical_path - if project_root.resolve() == Path.cwd().resolve(): + resolved = resolver.resolve_include(normalized.as_posix()) + if resolved == resolver.cwd / normalized: return os.path.join("./", normalized.as_posix()) return str(resolved) diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py index efe0ac30c..352a5366d 100644 --- a/pdd/sync_core/includes.py +++ b/pdd/sync_core/includes.py @@ -98,16 +98,6 @@ def digest(self) -> str: return hashlib.sha256(encoded).hexdigest() -_INCLUDE_PATTERN = re.compile( - r'\s[^>]*?)?(?' - r'\s*(?P[^<>\r\n]+?)\s*' - r'|\s[^>]*?)/>', -) -_INCLUDE_MANY_PATTERN = re.compile( - r'\s[^>]*+)?>(?P.*?)', - re.DOTALL, -) -_BACKTICK_PATTERN = re.compile(r"```<([^>]*+)>```") _ATTRIBUTE_PATTERN = re.compile(r'(\w+)\s*=\s*["\']([^"\']*)["\']') @@ -130,19 +120,56 @@ def _enabled(value: str | None) -> bool: } -def parse_include_references(text: str) -> tuple[IncludeReference, ...]: - """Parse includes once, preserving duplicates and deterministic source order.""" - if not text: - return () +def _tag_end(text: str, start: int) -> int: + """Return the exclusive end of an opening tag, or ``-1`` when incomplete.""" + return text.find(">", start) + + +def _is_tag_boundary(text: str, index: int) -> bool: + """Return whether *index* is a valid character after an include tag name.""" + return index == len(text) or text[index].isspace() or text[index] in ">/" + + +def _parse_xml_includes(text: str) -> list[IncludeReference]: + """Scan ```` markup without regex backtracking over user text.""" references: list[IncludeReference] = [] - for match in _INCLUDE_PATTERN.finditer(text): - attrs = _parse_attrs(match.group("attrs") or match.group("attrs_self") or "") - body = match.group("content") or "" - path = (attrs.get("path") or body).strip() + cursor = 0 + tag_name = "\r\n"): + cursor = opening_end + 1 + continue + path = (attrs.get("path") or body).strip() + cursor = close_start + len(close_tag) if path: references.append( IncludeReference( - match.start(), + start, path, IncludeSyntax.XML, attrs.get("select"), @@ -151,29 +178,73 @@ def parse_include_references(text: str) -> tuple[IncludeReference, ...]: _enabled(attrs.get("expand")), ) ) - for match in _INCLUDE_MANY_PATTERN.finditer(text): - attrs = _parse_attrs(match.group("attrs") or "") - values = ( - item.strip() - for line in match.group("inner").splitlines() - for item in line.split(",") - ) - for offset, path in enumerate(value for value in values if value): + + +def _parse_include_many(text: str) -> list[IncludeReference]: + """Scan ```` markup with a single forward cursor.""" + references: list[IncludeReference] = [] + cursor = 0 + tag_name = " list[IncludeReference]: + """Scan backtick include fences without regex matching on prompt text.""" + references: list[IncludeReference] = [] + cursor = 0 + opening = "```<" + closing = ">```" + while True: + start = text.find(opening, cursor) + if start < 0: + return references + path_end = text.find(closing, start + len(opening)) + if path_end < 0: + return references + path = text[start + len(opening):path_end].strip() if path: - references.append( - IncludeReference(match.start(), path, IncludeSyntax.BACKTICK) - ) + references.append(IncludeReference(start, path, IncludeSyntax.BACKTICK)) + cursor = path_end + len(closing) + + +def parse_include_references(text: str) -> tuple[IncludeReference, ...]: + """Parse includes once, preserving duplicates and deterministic source order.""" + if not text: + return () + references = _parse_xml_includes(text) + references.extend(_parse_include_many(text)) + references.extend(_parse_backtick_includes(text)) return tuple(sorted(references)) From 3ca27eda40b6f650bf1b16415808ba77a575f0a9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 18:36:39 -0700 Subject: [PATCH 009/128] test(sync): reproduce remaining CodeQL alerts --- tests/test_ci_drift_heal.py | 24 ++++++++++++++++-------- tests/test_sync_core_includes.py | 9 +++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/test_ci_drift_heal.py b/tests/test_ci_drift_heal.py index ac264040a..f999b37f2 100644 --- a/tests/test_ci_drift_heal.py +++ b/tests/test_ci_drift_heal.py @@ -4346,7 +4346,7 @@ def test_commit_and_push_stages_tracked_updates(self): class TestMainDryRunJson: def test_dry_run_json_outputs_report(self, capsys): - """--dry-run with --json prints the report as JSON and exits based on ok.""" + """--dry-run with --json emits only its stable summary schema.""" report = { "ok": True, "summary": { @@ -4361,7 +4361,11 @@ def test_dry_run_json_outputs_report(self, capsys): assert result == 0 out = capsys.readouterr().out parsed = json.loads(out) - assert parsed == report + assert parsed == { + "ok": True, + "consumer": "ci-heal", + "summary": report["summary"], + } def test_dry_run_json_returns_one_when_not_ok(self, capsys): report = { @@ -4377,9 +4381,9 @@ def test_dry_run_json_returns_one_when_not_ok(self, capsys): result = main(dry_run=True, as_json=True) assert result == 1 - def test_dry_run_json_redacts_secret_bearing_failure_data(self, capsys): - """The machine-readable dry-run report must not emit embedded secrets.""" - token = "ghp_" + "a" * 36 + def test_dry_run_json_excludes_exception_bearing_failure_data(self, capsys): + """The output sink cannot receive candidate-controlled failure values.""" + secret = "candidate exception text: ssh-private-key" report = { "ok": False, "summary": { @@ -4388,15 +4392,19 @@ def test_dry_run_json_redacts_secret_bearing_failure_data(self, capsys): "unbaselined": 0, "failures": 1, }, - "failures": [f"remote command failed: Authorization: Bearer {token}"], + "failures": [secret], } with patch("pdd.continuous_sync.build_report", return_value=report): result = main(dry_run=True, as_json=True) assert result == 1 output = capsys.readouterr().out - assert token not in output - assert "[REDACTED]" in output + assert secret not in output + assert json.loads(output) == { + "ok": False, + "consumer": "ci-heal", + "summary": report["summary"], + } class TestHealModuleConflict: diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py index 9e6fd8913..41b1ad589 100644 --- a/tests/test_sync_core_includes.py +++ b/tests/test_sync_core_includes.py @@ -68,6 +68,15 @@ def test_malformed_include_text_is_bounded() -> None: assert parse_include_references(text) == () +@pytest.mark.timeout(1) +def test_attribute_parser_is_bounded_for_repeated_word_characters() -> None: + """Unquoted attribute-like input must not cause quadratic regex retries.""" + text = "docs/a.md" + assert [reference.path for reference in parse_include_references(text)] == [ + "docs/a.md" + ] + + def test_preprocess_and_sync_order_use_the_same_parser(tmp_path) -> None: prompt = tmp_path / "widget_python.prompt" prompt.write_text(PROMPT, encoding="utf-8") From 9920af420c0e1cf47176e0f631633b061f0669a7 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 18:39:36 -0700 Subject: [PATCH 010/128] fix(sync): eliminate remaining CodeQL alerts --- pdd/ci_drift_heal.py | 30 ++++++++++++++++++--- pdd/sync_core/includes.py | 56 +++++++++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/pdd/ci_drift_heal.py b/pdd/ci_drift_heal.py index 1b80a8fbb..7b32413d8 100644 --- a/pdd/ci_drift_heal.py +++ b/pdd/ci_drift_heal.py @@ -25,8 +25,6 @@ from rich.console import Console from rich.table import Table -from pdd.context_snapshot import redact_snapshot_text - console = Console() _HEAL_SUBPROCESS_TIMEOUT_DEFAULT = 2400 @@ -60,6 +58,31 @@ def _heal_subprocess_timeout() -> int: _INVARIANT_KEYS = {"include", "pdd_tags", "percent_markers", "fenced_blocks"} +def _dry_run_json_summary(report: Dict[str, Any]) -> Dict[str, Any]: + """Return the fixed, value-free dry-run JSON schema for CI output.""" + summary = report.get("summary") + if not isinstance(summary, dict): + summary = {} + + def count(name: str) -> int: + value = summary.get(name) + valid_count = ( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + ) + return value if valid_count else 0 + + return { + "ok": report.get("ok") is True, + "consumer": "ci-heal", + "summary": { + "metadata_stale": count("metadata_stale"), + "conflicts": count("conflicts"), + "unbaselined": count("unbaselined"), + "failures": count("failures"), + }, + } + + # --------------------------------------------------------------------------- # Data model # --------------------------------------------------------------------------- @@ -1895,8 +1918,7 @@ def main( report = build_report(consumer="ci-heal", modules=modules) if as_json: - serialized = json.dumps(report, indent=2, sort_keys=True) - print(redact_snapshot_text(serialized)[0]) + print(json.dumps(_dry_run_json_summary(report), indent=2, sort_keys=True)) else: summary = report["summary"] console.print( diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py index 352a5366d..4c9949309 100644 --- a/pdd/sync_core/includes.py +++ b/pdd/sync_core/includes.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re import hashlib import json import posixpath @@ -98,14 +97,61 @@ def digest(self) -> str: return hashlib.sha256(encoded).hexdigest() -_ATTRIBUTE_PATTERN = re.compile(r'(\w+)\s*=\s*["\']([^"\']*)["\']') +def _is_attribute_name_char(character: str) -> bool: + """Return whether a character belongs to a Python ``\\w`` attribute name.""" + return character == "_" or character.isalnum() + + +def _has_boolean_attr(raw: str, name: str) -> bool: + """Find one bare boolean attribute with the legacy ASCII boundaries.""" + cursor = 0 + while True: + start = raw.find(name, cursor) + if start < 0: + return False + end = start + len(name) + before_is_word = start > 0 and ( + raw[start - 1].isascii() and _is_attribute_name_char(raw[start - 1]) + ) + after_is_word = end < len(raw) and ( + raw[end].isascii() and _is_attribute_name_char(raw[end]) + ) + if not before_is_word and not after_is_word: + return True + cursor = end def _parse_attrs(raw: str) -> dict[str, str]: - attrs = {match.group(1): match.group(2) for match in _ATTRIBUTE_PATTERN.finditer(raw)} + """Parse quoted and boolean include attributes with forward-only scans.""" + attrs: dict[str, str] = {} + cursor = 0 + while cursor < len(raw): + while cursor < len(raw) and not _is_attribute_name_char(raw[cursor]): + cursor += 1 + name_start = cursor + while cursor < len(raw) and _is_attribute_name_char(raw[cursor]): + cursor += 1 + if name_start == cursor: + break + name = raw[name_start:cursor] + while cursor < len(raw) and raw[cursor].isspace(): + cursor += 1 + if cursor == len(raw) or raw[cursor] != "=": + continue + cursor += 1 + while cursor < len(raw) and raw[cursor].isspace(): + cursor += 1 + if cursor == len(raw) or raw[cursor] not in "\"'": + continue + quote = raw[cursor] + value_start = cursor + 1 + value_end = raw.find(quote, value_start) + if value_end < 0: + break + attrs[name] = raw[value_start:value_end] + cursor = value_end + 1 for boolean_name in ("optional", "expand"): - pattern = rf"(? Date: Fri, 10 Jul 2026 19:09:38 -0700 Subject: [PATCH 011/128] test(sync): reproduce remaining foundation regressions --- tests/test_ci_drift_heal.py | 37 +++++++++++++++++++++++++++++++++++++ tests/test_preprocess.py | 11 +++++------ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/test_ci_drift_heal.py b/tests/test_ci_drift_heal.py index f999b37f2..525bcbcf8 100644 --- a/tests/test_ci_drift_heal.py +++ b/tests/test_ci_drift_heal.py @@ -4365,6 +4365,7 @@ def test_dry_run_json_outputs_report(self, capsys): "ok": True, "consumer": "ci-heal", "summary": report["summary"], + "units": [], } def test_dry_run_json_returns_one_when_not_ok(self, capsys): @@ -4404,8 +4405,44 @@ def test_dry_run_json_excludes_exception_bearing_failure_data(self, capsys): "ok": False, "consumer": "ci-heal", "summary": report["summary"], + "units": [], } + def test_dry_run_json_retains_value_free_unit_classifications(self, capsys): + """CI consumers can compare classifications without receiving paths or errors.""" + report = { + "ok": False, + "summary": { + "metadata_stale": 1, + "conflicts": 0, + "unbaselined": 0, + "failures": 0, + }, + "units": [ + { + "basename": "widget", + "language": "python", + "classification": "CODE_CHANGED", + "paths": {"code": "/candidate/private/widget.py"}, + "reason": "candidate-controlled diagnostic", + } + ], + } + with patch("pdd.continuous_sync.build_report", return_value=report): + result = main(dry_run=True, as_json=True) + + assert result == 1 + output = json.loads(capsys.readouterr().out) + assert output["units"] == [ + { + "basename": "widget", + "language": "python", + "classification": "CODE_CHANGED", + } + ] + assert "paths" not in output["units"][0] + assert "reason" not in output["units"][0] + class TestHealModuleConflict: def test_conflict_operation_returns_false(self): diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py index 6693a01e9..994ac1ba7 100644 --- a/tests/test_preprocess.py +++ b/tests/test_preprocess.py @@ -770,10 +770,10 @@ def test_get_file_path() -> None: path = get_file_path(filename) assert path == "./test.txt" - # Test with absolute path + # Legacy preprocessing accepts explicit local paths. Canonical sync graph + # construction enforces repository-relative managed paths separately. abs_path = "/absolute/path/test.txt" - with pytest.raises(ValueError, match="absolute include path"): - get_file_path(abs_path) + assert get_file_path(abs_path) == abs_path def test_preprocess_resolves_bundled_prompting_guide_outside_checkout( @@ -3405,10 +3405,9 @@ def test_include_many_optional_attribute_silences_missing(tmp_path, monkeypatch, def test_get_file_path_absolute_returned_unchanged(tmp_path) -> None: - """Absolute managed include paths are rejected by protected policy.""" + """Legacy preprocessing preserves explicit absolute local include paths.""" p = str(tmp_path / "abs.txt") - with pytest.raises(ValueError, match="absolute include path"): - get_file_path(p) + assert get_file_path(p) == p def test_web_tag_inside_code_block_not_processed() -> None: From 421ec3a87f95fa1e2b35c02a208877baf84d67bb Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 19:11:43 -0700 Subject: [PATCH 012/128] fix(sync): restore foundation compatibility boundaries --- pdd/ci_drift_heal.py | 17 ++++++++++++++++- pdd/preprocess.py | 12 ++++++++++-- pdd/sync_core/includes.py | 4 ++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pdd/ci_drift_heal.py b/pdd/ci_drift_heal.py index 7b32413d8..4ff31130b 100644 --- a/pdd/ci_drift_heal.py +++ b/pdd/ci_drift_heal.py @@ -59,7 +59,7 @@ def _heal_subprocess_timeout() -> int: def _dry_run_json_summary(report: Dict[str, Any]) -> Dict[str, Any]: - """Return the fixed, value-free dry-run JSON schema for CI output.""" + """Return the fixed dry-run schema without paths or diagnostic payloads.""" summary = report.get("summary") if not isinstance(summary, dict): summary = {} @@ -71,6 +71,20 @@ def count(name: str) -> int: ) return value if valid_count else 0 + units = report.get("units") + if not isinstance(units, list): + units = [] + projected_units = [] + for unit in units: + if not isinstance(unit, dict): + continue + identity = { + name: unit.get(name) + for name in ("basename", "language", "classification") + } + if all(isinstance(value, str) and value for value in identity.values()): + projected_units.append(identity) + return { "ok": report.get("ok") is True, "consumer": "ci-heal", @@ -80,6 +94,7 @@ def count(name: str) -> int: "unbaselined": count("unbaselined"), "failures": count("failures"), }, + "units": projected_units, } diff --git a/pdd/preprocess.py b/pdd/preprocess.py index e93e985ae..ee44068bc 100644 --- a/pdd/preprocess.py +++ b/pdd/preprocess.py @@ -433,10 +433,18 @@ def get_file_path(file_name: str) -> str: resolver = get_default_resolver() relpath = Path(file_name) if relpath.is_absolute(): - raise ValueError(f"absolute include path is not allowed: {file_name}") + return str(relpath) normalized = PurePosixPath(os.path.normpath(file_name).replace(os.sep, "/")) resolved = resolver.resolve_include(normalized.as_posix()) - if resolved == resolver.cwd / normalized: + cwd_candidate = resolver.cwd / normalized + if resolved == cwd_candidate: + return os.path.join("./", normalized.as_posix()) + try: + resolved.relative_to(resolver.cwd) + except ValueError: + pass + else: + # A nested source checkout must not shadow a missing active-project file. return os.path.join("./", normalized.as_posix()) return str(resolved) diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py index 4c9949309..917e86bb0 100644 --- a/pdd/sync_core/includes.py +++ b/pdd/sync_core/includes.py @@ -207,10 +207,10 @@ def _parse_xml_includes(text: str) -> list[IncludeReference]: cursor = opening_end + 1 continue body = text[opening_end + 1:close_start] - if any(character in body for character in "<>\r\n"): + path = (attrs.get("path") or body).strip() + if any(character in path for character in "<>\r\n"): cursor = opening_end + 1 continue - path = (attrs.get("path") or body).strip() cursor = close_start + len(close_tag) if path: references.append( From 068a7ecee9c4f3cebd92e7c61a79a118d056cce9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 22:43:55 -0700 Subject: [PATCH 013/128] test(sync): reproduce xhigh review findings --- tests/test_sync_core_cli.py | 12 +++- tests/test_sync_core_lifecycle_scenarios.py | 63 +++++++++++++++++++ tests/test_sync_core_runner.py | 67 +++++++++++++++++++++ 3 files changed, 139 insertions(+), 3 deletions(-) diff --git a/tests/test_sync_core_cli.py b/tests/test_sync_core_cli.py index 9448b0683..a41bd7780 100644 --- a/tests/test_sync_core_cli.py +++ b/tests/test_sync_core_cli.py @@ -5,8 +5,8 @@ from pdd.cli import cli -def test_documented_sync_certify_spelling_is_registered() -> None: - result = CliRunner().invoke(cli, ["sync", "certify", "--help"]) +def test_root_certify_command_is_registered() -> None: + result = CliRunner().invoke(cli, ["certify", "--help"]) assert result.exit_code == 0, result.output assert "--repos" in result.output assert "--run-lifecycle-matrix" in result.output @@ -17,7 +17,6 @@ def test_global_certify_requires_complete_acceptance_inputs(tmp_path) -> None: result = CliRunner().invoke( cli, [ - "sync", "certify", "--repos", "pdd,pdd_cloud", @@ -30,6 +29,13 @@ def test_global_certify_requires_complete_acceptance_inputs(tmp_path) -> None: assert "--full-inventory" in result.output +def test_sync_certify_remains_available_as_sync_basename() -> None: + result = CliRunner().invoke(cli, ["sync", "certify", "--dry-run", "--json"]) + assert result.exit_code == 0, result.output + assert "global certification requires" not in result.output + assert "--repos" not in result.output + + def test_reviewed_baseline_command_is_registered() -> None: result = CliRunner().invoke(cli, ["baseline", "--help"]) assert result.exit_code == 0, result.output diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index 84c8d2c8e..d6b34bd1c 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -1,5 +1,6 @@ """Process-level merge, wheel, and real-consumer lifecycle scenarios.""" +import argparse import os import json import subprocess @@ -8,6 +9,10 @@ import pytest +from pdd.sync_core.certificate import LifecycleResult +from pdd.sync_core.lifecycle import _isolated_lifecycle_environment +from pdd.sync_core.scenario_contract import REQUIRED_SCENARIOS +from pdd.sync_core import scenario_harness from pdd.sync_core import ( PlannedWrite, TransactionConflict, @@ -44,6 +49,64 @@ def _record_metric(name: str, value: int) -> None: path.write_text(json.dumps(payload, sort_keys=True)) +def test_lifecycle_contract_requires_public_repair_injection_scenarios() -> None: + required = { + "public-prompt-only-repair-zero-write-rerun", + "public-code-only-repair-zero-write-rerun", + "public-test-only-repair-zero-write-rerun", + "public-include-only-repair-zero-write-rerun", + "public-simultaneous-edit-repair-block", + } + assert required <= REQUIRED_SCENARIOS + + +def test_lifecycle_predicate_requires_dependency_environment_digest() -> None: + result = LifecycleResult( + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256="a" * 64, + dependency_environment_digest="b" * 64, + ) + assert result.dependency_environment_digest == "b" * 64 + + +def test_lifecycle_environment_strips_signer_capabilities(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("PDD_ATTESTATION_SIGNER_COMMAND", "protected-attestation") + monkeypatch.setenv("PDD_CERTIFICATE_ISSUER", "protected-issuer") + monkeypatch.setenv("PDD_RELEASED_CHECKER_COMMAND", "protected-checker") + environment = _isolated_lifecycle_environment(tmp_path) + assert "PDD_ATTESTATION_SIGNER_COMMAND" not in environment + assert "PDD_CERTIFICATE_ISSUER" not in environment + assert "PDD_RELEASED_CHECKER_COMMAND" not in environment + + +def test_candidate_scenario_environment_strips_signer_capabilities( + tmp_path, monkeypatch +) -> None: + captured = {} + + def fake_run(*_args, **kwargs): + captured.update(kwargs["env"]) + return subprocess.CompletedProcess(_args[0], 0, "", "") + + monkeypatch.setenv("PDD_ATTESTATION_SIGNER_COMMAND", "protected-attestation") + monkeypatch.setenv("PDD_CERTIFICATE_ISSUER", "protected-issuer") + monkeypatch.setenv("PDD_RELEASED_CHECKER_COMMAND", "protected-checker") + monkeypatch.setattr(scenario_harness.subprocess, "run", fake_run) + scenario_harness._candidate( + argparse.Namespace(candidate_python=sys.executable), + tmp_path, + "--version", + ) + assert "PDD_ATTESTATION_SIGNER_COMMAND" not in captured + assert "PDD_CERTIFICATE_ISSUER" not in captured + assert "PDD_RELEASED_CHECKER_COMMAND" not in captured + + def test_merge_base_movement_blocks_stale_repair_and_converges(tmp_path) -> None: """A repair planned before merge movement must never overwrite merged bytes.""" root = tmp_path / "merge-repo" diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 3e9420644..392559d6a 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -1,5 +1,6 @@ """Tests for pass-only trusted runner normalization and self-certification guards.""" +import json import subprocess from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -137,6 +138,49 @@ def test_candidate_modified_imported_test_helper_cannot_self_certify(tmp_path) - "from tests.helper import expected\ndef test_widget(): assert expected() == 1\n", ) (root / "tests/__init__.py").write_text("") + (root / "tests/helper.py").write_text("def expected():\n return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "tests/helper.py" in executions[0].detail + + +def test_candidate_modified_importfrom_alias_helper_cannot_self_certify(tmp_path) -> None: + root, _initial = _repository( + tmp_path, + "from tests import helper\ndef test_widget(): assert helper.expected() == 1\n", + ) + (root / "tests/__init__.py").write_text("") + (root / "tests/helper.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "base helper") + base = _git(root, "rev-parse", "HEAD") + (root / "tests/helper.py").write_text("def expected():\n return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "tests/helper.py" in executions[0].detail + + +def test_candidate_modified_relative_importfrom_helper_cannot_self_certify(tmp_path) -> None: + root = tmp_path / "repo" + root.mkdir() + _git(root, "init", "-q") + _git(root, "config", "user.email", "runner@example.com") + _git(root, "config", "user.name", "Runner Test") + (root / "tests").mkdir() + (root / "tests/__init__.py").write_text("") + (root / "tests/test_widget.py").write_text( + "from . import helper\ndef test_widget(): assert helper.expected() == 1\n" + ) + (root / "tests/helper.py").write_text("def expected(): return 2\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "base helper") + base = _git(root, "rev-parse", "HEAD") (root / "tests/helper.py").write_text("def expected(): return 1\n") _git(root, "add", ".") _git(root, "commit", "-q", "-m", "candidate helper") @@ -160,6 +204,29 @@ def test_validator_subprocess_cannot_read_signing_secret(tmp_path, monkeypatch) assert (root / "observed-secret").read_text() == "missing" +def test_validator_subprocess_cannot_read_signer_capabilities(tmp_path, monkeypatch) -> None: + content = ( + "import json, os\nfrom pathlib import Path\n" + "def test_widget():\n" + " Path('observed-capabilities.json').write_text(json.dumps({\n" + " 'attestation_command': os.environ.get('PDD_ATTESTATION_SIGNER_COMMAND'),\n" + " 'certificate_issuer': os.environ.get('PDD_CERTIFICATE_ISSUER'),\n" + " 'released_checker': os.environ.get('PDD_RELEASED_CHECKER_COMMAND'),\n" + " }, sort_keys=True))\n" + ) + root, head = _repository(tmp_path, content) + monkeypatch.setenv("PDD_ATTESTATION_SIGNER_COMMAND", "protected-attestation") + monkeypatch.setenv("PDD_CERTIFICATE_ISSUER", "protected-issuer") + monkeypatch.setenv("PDD_RELEASED_CHECKER_COMMAND", "protected-checker") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + assert json.loads((root / "observed-capabilities.json").read_text()) == { + "attestation_command": None, + "certificate_issuer": None, + "released_checker": None, + } + + def test_ambient_pytest_options_and_plugins_are_disabled(tmp_path, monkeypatch) -> None: content = ( "import os\n" From b9245a9d1f77288374670413c78954594583a1a9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 22:56:50 -0700 Subject: [PATCH 014/128] fix(sync): harden xhigh certification boundaries --- pdd/core/cli.py | 6 --- pdd/sync_core/certificate.py | 25 +++++++---- pdd/sync_core/isolation.py | 57 +++++++++++++++++++++++- pdd/sync_core/lifecycle.py | 67 +++++++++++++++++++++-------- pdd/sync_core/released_checker.py | 2 +- pdd/sync_core/runner.py | 24 ++++++----- pdd/sync_core/scenario_contract.py | 5 +++ pdd/sync_core/scenario_harness.py | 23 +++++----- tests/test_sync_core_certificate.py | 24 ++++++++++- 9 files changed, 176 insertions(+), 57 deletions(-) diff --git a/pdd/core/cli.py b/pdd/core/cli.py index 67ef1994e..b387b684f 100644 --- a/pdd/core/cli.py +++ b/pdd/core/cli.py @@ -350,12 +350,6 @@ def _write_result_core_dump( class PDDCLI(click.Group): """Custom Click Group that adds a Generate Suite section to root help.""" - def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: - """Reserve ``sync certify`` as the global certificate command spelling.""" - if len(args) >= 2 and args[0:2] == ["sync", "certify"]: - args = ["certify", *args[2:]] - return super().parse_args(ctx, args) - def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: self.format_usage(ctx, formatter) formatter.write_text(PDD_FULL_TAGLINE) diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index f6ffdceec..f45f7fd3a 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -87,6 +87,7 @@ class LifecycleResult: post_merge_tree_changes: int missing_scenarios: tuple[str, ...] = () candidate_wheel_sha256: str = "" + dependency_environment_digest: str = "" @dataclass(frozen=True) @@ -396,6 +397,7 @@ def _complete_nightly( "collection_errors", "timeouts", "candidate_wheel_sha256", + "dependency_environment_digest", } return ( required_counts <= counts.keys() @@ -527,6 +529,11 @@ def _scan_predicate( character in "0123456789abcdef" for character in lifecycle.candidate_wheel_sha256 ) + and len(lifecycle.dependency_environment_digest) == 64 + and all( + character in "0123456789abcdef" + for character in lifecycle.dependency_environment_digest + ) and extra["nightly_observation_complete"] == 1 ) @@ -653,20 +660,19 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] for name in lifecycle_names ) or not isinstance(lifecycle_payload.get("missing_scenarios"), list): return False, False - candidate_wheel_sha256 = lifecycle_payload.get("candidate_wheel_sha256") - if ( - not isinstance(candidate_wheel_sha256, str) - or len(candidate_wheel_sha256) != 64 - or any( - character not in "0123456789abcdef" - for character in candidate_wheel_sha256 - ) + digest_names = ("candidate_wheel_sha256", "dependency_environment_digest") + digests = {name: lifecycle_payload.get(name) for name in digest_names} + if any( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + for value in digests.values() ): return False, False lifecycle = LifecycleResult( *(lifecycle_payload[name] for name in lifecycle_names), tuple(lifecycle_payload["missing_scenarios"]), - candidate_wheel_sha256, + *(digests[name] for name in digest_names), ) extra_names = { "pdd_cloud_vendored_sync_semantics", @@ -791,6 +797,7 @@ def _build_global_certificate_from_targets( "post_merge_tree_changes": lifecycle.post_merge_tree_changes, "missing_scenarios": list(lifecycle.missing_scenarios), "candidate_wheel_sha256": lifecycle.candidate_wheel_sha256, + "dependency_environment_digest": lifecycle.dependency_environment_digest, }, } body["scan_ok"] = all(report.get("ok") for report in reports) and _scan_predicate( diff --git a/pdd/sync_core/isolation.py b/pdd/sync_core/isolation.py index 789f11fcb..0ea7cd3ce 100644 --- a/pdd/sync_core/isolation.py +++ b/pdd/sync_core/isolation.py @@ -1,4 +1,9 @@ -"""Shared credential-removal policy for untrusted child processes.""" +"""Shared environment policy for untrusted child processes.""" + +from __future__ import annotations + +import os +from pathlib import Path SECRET_ENV_MARKERS = ( "API_KEY", @@ -8,3 +13,53 @@ "SIGNING_KEY", "TOKEN", ) + +SAFE_UNTRUSTED_ENV_KEYS = frozenset( + { + "CI", + "HOME", + "LANG", + "LC_ALL", + "PATH", + "PATHEXT", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "USERPROFILE", + "WINDIR", + } +) + +BLOCKED_CAPABILITY_TOKENS = ( + "ATTESTATION", + "CERTIFICATE", + "SIGNER", + "RELEASED_CHECKER", +) + + +def untrusted_child_environment( + *, + home: Path | None = None, + extra: dict[str, str] | None = None, + drop: set[str] | None = None, +) -> dict[str, str]: + """Return a small environment for candidate-authored code.""" + environment = { + key: value + for key, value in os.environ.items() + if key.upper() in SAFE_UNTRUSTED_ENV_KEYS + and not any(marker in key.upper() for marker in SECRET_ENV_MARKERS) + and not any(token in key.upper() for token in BLOCKED_CAPABILITY_TOKENS) + } + if home is not None: + environment["HOME"] = str(home) + environment["XDG_CONFIG_HOME"] = str(home / ".config") + environment["XDG_CACHE_HOME"] = str(home / ".cache") + for key in drop or set(): + environment.pop(key, None) + environment.update(extra or {}) + return environment diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index f323c0df7..2ff4d2ab1 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -12,23 +12,17 @@ from typing import Any from .certificate import LifecycleResult -from .isolation import SECRET_ENV_MARKERS +from .isolation import untrusted_child_environment from .scenario_contract import REQUIRED_SCENARIOS def _isolated_lifecycle_environment(home: Path) -> dict[str, str]: """Build a credential-free environment with no source import overrides.""" - environment = { - key: value - for key, value in os.environ.items() - if not any(marker in key.upper() for marker in SECRET_ENV_MARKERS) - and key not in {"PYTHONPATH", "PYTHONHOME", "PDD_PATH"} - } - environment["HOME"] = str(home) - environment["XDG_CONFIG_HOME"] = str(home / ".config") - environment["XDG_CACHE_HOME"] = str(home / ".cache") - environment["PYTHONNOUSERSITE"] = "1" - return environment + return untrusted_child_environment( + home=home, + extra={"PYTHONNOUSERSITE": "1"}, + drop={"PYTHONPATH", "PYTHONHOME", "PDD_PATH"}, + ) def _failed_result(*, timeout: bool = False) -> LifecycleResult: @@ -41,6 +35,7 @@ def _failed_result(*, timeout: bool = False) -> LifecycleResult: 1, tuple(sorted(REQUIRED_SCENARIOS)), "", + "", ) @@ -78,7 +73,40 @@ def _normalized_results(payload: Any) -> dict[str, dict[str, Any]]: return results -def _install_candidate_wheel(temporary: Path, home: Path, wheel: Path) -> Path | None: +def _dependency_environment_digest(candidate_python: Path, isolated: dict[str, str]) -> str: + """Hash the installed distribution set visible to the candidate wheel.""" + probe = subprocess.run( + [ + str(candidate_python), + "-c", + ( + "import importlib.metadata as m, json;" + "rows=[];" + "\nfor d in m.distributions():" + "\n rows.append({" + "'name': d.metadata.get('Name', '')," + "'version': d.version," + "'files': sorted(" + "(str(f), getattr(getattr(f, 'hash', None), 'value', ''), f.size or 0)" + " for f in (d.files or ()))" + "})" + "\nprint(json.dumps(sorted(rows, key=lambda r: (r['name'].lower(), r['version']))," + " sort_keys=True, separators=(',', ':')))" + ), + ], + capture_output=True, + text=True, + check=False, + env=isolated, + ) + if probe.returncode != 0: + return "" + return hashlib.sha256(probe.stdout.encode()).hexdigest() + + +def _install_candidate_wheel( + temporary: Path, home: Path, wheel: Path +) -> tuple[Path, str] | None: """Install the exact candidate wheel in a separate isolated environment.""" environment = temporary / "candidate-venv" isolated = _isolated_lifecycle_environment(home) @@ -100,6 +128,7 @@ def _install_candidate_wheel(temporary: Path, home: Path, wheel: Path) -> Path | "-m", "pip", "install", + "--no-deps", "--only-binary=:all:", "--disable-pip-version-check", "--force-reinstall", @@ -110,7 +139,9 @@ def _install_candidate_wheel(temporary: Path, home: Path, wheel: Path) -> Path | check=False, env=isolated, ) - return candidate_python if installed.returncode == 0 else None + if installed.returncode != 0: + return None + return candidate_python, _dependency_environment_digest(candidate_python, isolated) def run_lifecycle_matrix( @@ -122,7 +153,7 @@ def run_lifecycle_matrix( cloud_head_ref: str | None = None, timeout_seconds: int = 1200, ) -> LifecycleResult: - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-locals """Run only the scenario harness installed with the released checker.""" del root # Candidate repository tests are never lifecycle evidence. if ( @@ -137,11 +168,12 @@ def run_lifecycle_matrix( temporary = Path(directory) output = temporary / "result.json" (temporary / "home").mkdir(mode=0o700) - candidate_python = _install_candidate_wheel( + installed_candidate = _install_candidate_wheel( temporary, temporary / "home", Path(candidate_wheel) ) - if candidate_python is None: + if installed_candidate is None: return _failed_result() + candidate_python, dependency_digest = installed_candidate command = [ sys.executable, "-I", @@ -198,4 +230,5 @@ def run_lifecycle_matrix( sum(int(row["post_merge_tree_changes"]) for row in results.values()), missing, hashlib.sha256(Path(candidate_wheel).read_bytes()).hexdigest(), + dependency_digest, ) diff --git a/pdd/sync_core/released_checker.py b/pdd/sync_core/released_checker.py index b152ae108..ae5aab550 100644 --- a/pdd/sync_core/released_checker.py +++ b/pdd/sync_core/released_checker.py @@ -93,7 +93,7 @@ def main() -> None: cli = importlib.import_module("pdd.cli").cli cli.main( - args=["sync", "certify", *sys.argv[1:]], + args=["certify", *sys.argv[1:]], prog_name="pdd-sync-checker", standalone_mode=True, ) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 765a751a0..b55568cca 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -5,7 +5,6 @@ import hashlib import ast import json -import os import re import subprocess import sys @@ -21,7 +20,7 @@ AttestationIssuer, AttestationRequest, ) -from .isolation import SECRET_ENV_MARKERS +from .isolation import untrusted_child_environment from .git_io import read_git_blob from .types import ( EvidenceOutcome, @@ -91,9 +90,16 @@ def _local_module_paths( for node in ast.walk(tree): if isinstance(node, ast.Import): modules.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: + elif isinstance(node, ast.ImportFrom): prefix = "." * node.level - modules.add(prefix + node.module) + module = prefix + (node.module or "") + if node.module: + modules.add(module) + for alias in node.names: + if alias.name == "*": + continue + separator = "" if not module or module.endswith(".") else "." + modules.add(f"{module}{separator}{alias.name}") elif isinstance(node, ast.Assign) and any( isinstance(target, ast.Name) and target.id == "pytest_plugins" for target in node.targets @@ -350,12 +356,10 @@ def _junit_outcome( def _pytest_environment() -> dict[str, str]: """Return the protected credential-free pytest process environment.""" - return { - key: value - for key, value in os.environ.items() - if not any(marker in key.upper() for marker in SECRET_ENV_MARKERS) - and key not in {"PYTEST_ADDOPTS", "PYTHONPATH"} - } | {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PYTHONNOUSERSITE": "1"} + return untrusted_child_environment( + extra={"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PYTHONNOUSERSITE": "1"}, + drop={"PYTEST_ADDOPTS", "PYTHONPATH"}, + ) def _run_test_node( diff --git a/pdd/sync_core/scenario_contract.py b/pdd/sync_core/scenario_contract.py index faa9ac259..0ecf3fb0c 100644 --- a/pdd/sync_core/scenario_contract.py +++ b/pdd/sync_core/scenario_contract.py @@ -13,6 +13,11 @@ "built-wheel-clean-environment", "candidate-wheel-public-report", "candidate-wheel-transaction-recovery", + "public-code-only-repair-zero-write-rerun", + "public-include-only-repair-zero-write-rerun", + "public-prompt-only-repair-zero-write-rerun", + "public-simultaneous-edit-repair-block", + "public-test-only-repair-zero-write-rerun", "pdd-cloud-real-consumer-canary", "released-checker-owned-scenario-harness", } diff --git a/pdd/sync_core/scenario_harness.py b/pdd/sync_core/scenario_harness.py index 3070b6fc5..5f37636d9 100644 --- a/pdd/sync_core/scenario_harness.py +++ b/pdd/sync_core/scenario_harness.py @@ -17,6 +17,7 @@ from .certificate import count_vendored_sync_semantics from .classifier import classify from .identity import initialize_repository_identity +from .isolation import untrusted_child_environment from .manifest import build_unit_manifest from .runner import AttestationIssue, RunBinding, RunnerConfig, run_profile from .scenario_contract import REQUIRED_SCENARIOS @@ -140,15 +141,7 @@ def _candidate( capture_output=True, text=True, check=False, - env={ - key: value - for key, value in os.environ.items() - if not any( - marker in key.upper() - for marker in ("API_KEY", "CREDENTIAL", "PASSWORD", "SECRET", "TOKEN") - ) - and key not in {"PYTHONPATH", "PYTHONHOME", "PDD_PATH"} - }, + env=untrusted_child_environment(drop={"PYTHONPATH", "PYTHONHOME", "PDD_PATH"}), ) @@ -807,9 +800,17 @@ def _owned_harness(_arguments: argparse.Namespace) -> ScenarioResult: if any( marker in key.upper() for key in os.environ - for marker in ("SIGNING_KEY", "CERTIFICATE_SIGNING", "ATTESTATION_SIGNING") + for marker in ( + "SIGNING_KEY", + "CERTIFICATE_SIGNING", + "ATTESTATION_SIGNING", + "SIGNER", + "CERTIFICATE", + "ATTESTATION", + "RELEASED_CHECKER", + ) ): - raise AssertionError("scenario child received signing material") + raise AssertionError("scenario child received signing capability") return ScenarioResult("released-checker-owned-scenario-harness", "PASS") diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 7f2044b03..4b977835a 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -30,6 +30,7 @@ ) OBSERVATION = NightlyObservation(True, True, 0, True, "BLOCKED", 0) CANDIDATE_WHEEL_SHA256 = "d" * 64 +DEPENDENCY_ENVIRONMENT_DIGEST = "e" * 64 @pytest.fixture(autouse=True) @@ -110,6 +111,7 @@ def _nightly(path, signer, count=7, *, include_observation=True): "post_merge_tree_changes": 0, "missing_scenarios": [], "candidate_wheel_sha256": CANDIDATE_WHEEL_SHA256, + "dependency_environment_digest": DEPENDENCY_ENVIRONMENT_DIGEST, } body = { "schema_version": 2, @@ -211,7 +213,14 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 ) lifecycle = LifecycleResult( - 0, 0, 0, 0, 0, 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256 + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, ) certificate = build_global_certificate( ( @@ -288,7 +297,15 @@ def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 ) lifecycle = LifecycleResult( - 0, 0, 0, 0, 0, 0, ("merge-group",), CANDIDATE_WHEEL_SHA256 + 0, + 0, + 0, + 0, + 0, + 0, + ("merge-group",), + CANDIDATE_WHEEL_SHA256, + DEPENDENCY_ENVIRONMENT_DIGEST, ) certificate = build_global_certificate( ( @@ -353,6 +370,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( 0, 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, ), nightly, checker_identity=CHECKER, @@ -506,6 +524,7 @@ def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( 0, 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, ), nightly, checker_identity=CHECKER, @@ -551,6 +570,7 @@ def test_signed_nightly_rows_without_observations_do_not_extend_streak( 0, 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, ), nightly, checker_identity=CHECKER, From b6cedd54491a5fcc530550a2791d0e0e6c8f3365 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 23:30:56 -0700 Subject: [PATCH 015/128] test(sync): reproduce xhigh re-review findings --- tests/test_get_language.py | 16 +++++++++++++ tests/test_sync_main.py | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/tests/test_get_language.py b/tests/test_get_language.py index 133ecaaab..f33c055bf 100644 --- a/tests/test_get_language.py +++ b/tests/test_get_language.py @@ -146,3 +146,19 @@ def test_js_ts_have_explicit_test_commands(self, language_format_path): assert js_cmd == 'node {file}', f"JavaScript run_test_command should be 'node {{file}}', got: {js_cmd}" assert ts_cmd == 'npx tsx {file}', f"TypeScript run_test_command should be 'npx tsx {{file}}', got: {ts_cmd}" + + +@pytest.mark.parametrize( + ("extension", "language"), + [ + (".sh", "Shell"), + (".m", "MATLAB"), + ("sh", "Shell"), + ], +) +def test_get_language_preserves_legacy_ambiguous_first_match(extension, language): + assert get_language(extension) == language + + +def test_get_language_unknown_extension_returns_empty_string(): + assert get_language(".not-a-real-extension") == "" diff --git a/tests/test_sync_main.py b/tests/test_sync_main.py index 3b9dc9d77..b63ae6f31 100644 --- a/tests/test_sync_main.py +++ b/tests/test_sync_main.py @@ -1627,6 +1627,55 @@ def fake_codegen(*_args, **_kwargs): # After sync_main returns, the prior outer value is restored. assert os.environ.get("PDD_REPAIR_DIRECTIVE") == "STALE-OUTER" + @pytest.mark.timeout(30) + def test_one_session_canonical_mode_blocks_before_writers( + self, mock_project_dir, mock_construct_paths, monkeypatch + ): + monkeypatch.setenv("PDD_SYNC_PROTECTED_BASE_SHA", "base-sha") + (mock_project_dir / "prompts" / "tinymod_python.prompt").write_text( + "% generate a tiny module" + ) + + fake_decision = MagicMock() + fake_decision.operation = "generate" + fake_pdd_files = { + "prompt": mock_project_dir / "prompts" / "tinymod_python.prompt", + "code": mock_project_dir / "generated" / "tinymod.py", + } + + with patch( + "pdd.sync_main.get_pdd_file_paths", + return_value=fake_pdd_files, + ), patch( + "pdd.sync_determine_operation.sync_determine_operation", + return_value=fake_decision, + ), patch( + "pdd.continuous_sync.canonical_sync_enabled", + return_value=True, + ), patch( + "pdd.code_generator_main.code_generator_main", + ) as mock_codegen, patch( + "pdd.one_session_sync.run_one_session_sync", + ) as mock_one_session: + ctx = create_mock_context({"local": True}) + with pytest.raises(RuntimeError, match="protected canonical sync blocks"): + sync_main( + ctx, + "tinymod", + max_attempts=1, + budget=1.0, + skip_verify=False, + skip_tests=False, + target_coverage=90.0, + dry_run=False, + one_session=True, + ) + + mock_codegen.assert_not_called() + mock_one_session.assert_not_called() + assert not fake_pdd_files["code"].parent.exists() + assert not fake_pdd_files["code"].exists() + # --- Tests for the two early-out sync_result branches (#1103) --- From 5bdd7a3c2ed82a933c5434e765980350eee7fa04 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 23:40:28 -0700 Subject: [PATCH 016/128] fix(sync): close xhigh re-review gaps --- pdd/get_language.py | 35 ++++++++++++++++++++++++++++++++--- pdd/sync_main.py | 11 +++++++++-- tests/test_sync_main.py | 26 ++++++++++++++------------ 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/pdd/get_language.py b/pdd/get_language.py index 06ca937a9..3b113bed5 100644 --- a/pdd/get_language.py +++ b/pdd/get_language.py @@ -1,7 +1,35 @@ """Compatibility language lookup backed by the protected bundled registry.""" +import csv +from pathlib import Path + from pdd.sync_core import LanguageRegistry, LanguageRegistryError +_LEGACY_UNKNOWN_EXTENSIONS = {".prompt"} + + +def _normalize_extension(extension: str) -> str: + normalized = extension.strip().casefold() + if normalized and not normalized.startswith("."): + normalized = "." + normalized + return normalized + + +def _legacy_first_match(extension: str) -> str: + if extension in _LEGACY_UNKNOWN_EXTENSIONS: + return "" + csv_path = Path(__file__).parent / "data" / "language_format.csv" + try: + with csv_path.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle): + row_extension = _normalize_extension(row.get("extension") or "") + if row_extension == extension: + return (row.get("language") or "").strip() + except (OSError, csv.Error): + return "" + return "" + + def get_language(extension: str) -> str: """ Determines the programming language associated with a given file extension. @@ -15,9 +43,10 @@ def get_language(extension: str) -> str: Raises: ValueError: If PDD_PATH environment variable is not set. """ - if not extension: + normalized = _normalize_extension(extension) + if not normalized: return "" try: - return LanguageRegistry.bundled().resolve_extension(extension).display_name + return LanguageRegistry.bundled().resolve_extension(normalized).display_name except LanguageRegistryError: - return "" + return _legacy_first_match(normalized) diff --git a/pdd/sync_main.py b/pdd/sync_main.py index 8ca29b5a1..73a5c3573 100644 --- a/pdd/sync_main.py +++ b/pdd/sync_main.py @@ -23,8 +23,6 @@ _find_pddrc_file, _get_relative_basename, _load_pddrc_config, - _detect_context, - _get_context_config, get_extension ) from .sync_determine_operation import get_pdd_file_paths, AmbiguousModuleError @@ -934,6 +932,15 @@ def sync_main( context_override=context_override, ) + from .continuous_sync import canonical_sync_enabled + + if canonical_sync_enabled(Path(pdd_files.get("prompt") or Path.cwd())): + raise RuntimeError( + "protected canonical sync blocks legacy production mutation; " + "use read-only reporting or trusted finalization until the " + "staged repair executor is enabled" + ) + # Check fingerprint — skip if module is already fully synced _one_session_skipped = False if not force: diff --git a/tests/test_sync_main.py b/tests/test_sync_main.py index b63ae6f31..6c51c4c0c 100644 --- a/tests/test_sync_main.py +++ b/tests/test_sync_main.py @@ -1658,21 +1658,23 @@ def test_one_session_canonical_mode_blocks_before_writers( "pdd.one_session_sync.run_one_session_sync", ) as mock_one_session: ctx = create_mock_context({"local": True}) - with pytest.raises(RuntimeError, match="protected canonical sync blocks"): - sync_main( - ctx, - "tinymod", - max_attempts=1, - budget=1.0, - skip_verify=False, - skip_tests=False, - target_coverage=90.0, - dry_run=False, - one_session=True, - ) + results, _cost, _model = sync_main( + ctx, + "tinymod", + max_attempts=1, + budget=1.0, + skip_verify=False, + skip_tests=False, + target_coverage=90.0, + dry_run=False, + one_session=True, + ) mock_codegen.assert_not_called() mock_one_session.assert_not_called() + lang_result = results["results_by_language"]["python"] + assert lang_result["success"] is False + assert "protected canonical sync blocks" in lang_result["error"] assert not fake_pdd_files["code"].parent.exists() assert not fake_pdd_files["code"].exists() From 0be869c9e1c4d635ba0380cacd28f4e244f07b61 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:18:08 -0700 Subject: [PATCH 017/128] fix: harden sync certification harness --- pdd/sync_core/includes.py | 17 +++++---- pdd/sync_core/runner.py | 38 ++++++++++++++++++--- pdd/sync_core/scenario_harness.py | 2 -- tests/test_sync_core_includes.py | 10 ++++++ tests/test_sync_core_lifecycle_scenarios.py | 20 +++++++++++ tests/test_sync_core_runner.py | 18 ++++++++++ 6 files changed, 93 insertions(+), 12 deletions(-) diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py index 917e86bb0..57f1526f4 100644 --- a/pdd/sync_core/includes.py +++ b/pdd/sync_core/includes.py @@ -385,13 +385,18 @@ def _resolved_targets( wildcard = any(character in reference.path for character in "*?[") for candidate in candidates: if wildcard: - matches = tuple( - sorted( - PurePosixPath(path.relative_to(policy.checkout_root).as_posix()) - for path in policy.checkout_root.glob(candidate.as_posix()) - if path.is_file() + try: + matches = tuple( + sorted( + PurePosixPath(path.relative_to(policy.checkout_root).as_posix()) + for path in policy.checkout_root.glob(candidate.as_posix()) + if path.is_file() + ) ) - ) + except ValueError as exc: + raise IncludeGraphError( + f"include wildcard pattern is invalid: {reference.path}" + ) from exc if matches: return matches continue diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index b55568cca..309f82340 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -39,6 +39,7 @@ PurePosixPath("setup.cfg"), ) PYTEST_PROTECTED_FLAGS = ("--strict-config", "--strict-markers", "-ra") +_CHECKER_PYTEST_PROBE = Path(__file__).with_name("pytest_probe.py").resolve() @dataclass(frozen=True) @@ -262,7 +263,7 @@ def runner_identity_digest( "--strict-config", "--strict-markers", "-p", - "pdd.sync_core.pytest_probe", + "", "", ], "pytest_environment": {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, @@ -362,6 +363,30 @@ def _pytest_environment() -> dict[str, str]: ) +def _trusted_probe_plugin(directory: Path) -> tuple[str, Path]: + """Create a unique plugin shim that loads the checker-owned probe by path.""" + plugin_name = "pdd_checker_pytest_probe" + plugin = directory / f"{plugin_name}.py" + plugin.write_text( + "\n".join( + ( + "import importlib.util", + "_SPEC = importlib.util.spec_from_file_location(", + f" {plugin_name!r}, {json.dumps(str(_CHECKER_PYTEST_PROBE))}", + ")", + "if _SPEC is None or _SPEC.loader is None:", + " raise ImportError('checker pytest probe is unavailable')", + "_MODULE = importlib.util.module_from_spec(_SPEC)", + "_SPEC.loader.exec_module(_MODULE)", + "pytest_collection_modifyitems = _MODULE.pytest_collection_modifyitems", + "", + ) + ), + encoding="utf-8", + ) + return plugin_name, directory + + def _run_test_node( root: Path, node_id: str, @@ -413,7 +438,9 @@ def _collect_node_ids( ) -> tuple[RunnerExecution, tuple[str, ...]]: """Collect exact pytest node IDs through the protected adapter.""" with tempfile.TemporaryDirectory(prefix="pdd-trusted-collection-") as directory: - collection_output = Path(directory) / "node-ids.json" + temporary = Path(directory) + collection_output = temporary / "node-ids.json" + plugin_name, plugin_directory = _trusted_probe_plugin(temporary) command = [ sys.executable, "-m", @@ -423,7 +450,7 @@ def _collect_node_ids( "--strict-config", "--strict-markers", "-p", - "pdd.sync_core.pytest_probe", + plugin_name, path.as_posix(), ] digest = hashlib.sha256( @@ -438,7 +465,10 @@ def _collect_node_ids( check=False, timeout=timeout_seconds, env=_pytest_environment() - | {"PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output)}, + | { + "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), + "PYTHONPATH": str(plugin_directory), + }, ) except subprocess.TimeoutExpired: return ( diff --git a/pdd/sync_core/scenario_harness.py b/pdd/sync_core/scenario_harness.py index 5f37636d9..662ee0fa1 100644 --- a/pdd/sync_core/scenario_harness.py +++ b/pdd/sync_core/scenario_harness.py @@ -691,7 +691,6 @@ def _candidate_public_report(arguments: argparse.Namespace) -> ScenarioResult: result = _candidate( arguments, root, - "sync", "certify", "--base-ref", commit, @@ -766,7 +765,6 @@ def _cloud_canary(arguments: argparse.Namespace) -> ScenarioResult: candidate = _candidate( arguments, cloud, - "sync", "certify", "--base-ref", arguments.cloud_base_ref, diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py index 41b1ad589..b18460d87 100644 --- a/tests/test_sync_core_includes.py +++ b/tests/test_sync_core_includes.py @@ -207,6 +207,16 @@ def test_include_many_glob_hashes_every_matching_file(tmp_path) -> None: ] +def test_invalid_wildcard_include_is_unit_scoped_error(tmp_path) -> None: + prompt = tmp_path / "prompts/widget.prompt" + prompt.parent.mkdir() + prompt.write_text("tests/**.yaml", encoding="utf-8") + with pytest.raises(IncludeGraphError, match="wildcard pattern is invalid"): + build_include_closure( + PurePosixPath("prompts/widget.prompt"), PathPolicy(tmp_path) + ) + + def test_prompt_tree_projects_include_to_generated_project_tree(tmp_path) -> None: prompt = tmp_path / "prompts/frontend/app/widget.prompt" config = tmp_path / "frontend/tsconfig.json" diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index d6b34bd1c..b4333558a 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -107,6 +107,26 @@ def fake_run(*_args, **kwargs): assert "PDD_RELEASED_CHECKER_COMMAND" not in captured +def test_lifecycle_public_report_uses_root_certify(monkeypatch) -> None: + observed = {} + + def fake_candidate(_arguments, _root, *command): + observed["command"] = command + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps({"counts": {"unbaselined": 1}}), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 1, "", "") + + monkeypatch.setattr(scenario_harness, "_candidate", fake_candidate) + result = scenario_harness._candidate_public_report( + argparse.Namespace(candidate_python=sys.executable) + ) + assert result.status == "PASS" + assert observed["command"][0] == "certify" + + def test_merge_base_movement_blocks_stale_repair_and_converges(tmp_path) -> None: """A repair planned before merge movement must never overwrite merged bytes.""" root = tmp_path / "merge-repo" diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 392559d6a..fc16d83e6 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -240,6 +240,24 @@ def test_ambient_pytest_options_and_plugins_are_disabled(tmp_path, monkeypatch) assert executions[0].outcome is EvidenceOutcome.PASS +def test_collection_probe_is_checker_owned_not_candidate_shadow(tmp_path) -> None: + root, head = _repository(tmp_path, "def test_widget(): assert True\n") + candidate_probe = root / "pdd/sync_core/pytest_probe.py" + candidate_probe.parent.mkdir(parents=True) + (root / "pdd/__init__.py").write_text("", encoding="utf-8") + (root / "pdd/sync_core/__init__.py").write_text("", encoding="utf-8") + candidate_probe.write_text( + "from pathlib import Path\n" + "def pytest_collection_modifyitems(items):\n" + " Path('candidate-probe-loaded').write_text('shadowed')\n" + " items[:] = []\n", + encoding="utf-8", + ) + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + assert not (root / "candidate-probe-loaded").exists() + + def test_deselected_declared_test_cannot_pass(tmp_path) -> None: content = "def test_keep(): assert True\ndef test_drop(): assert True\n" root, head = _repository(tmp_path, content) From 55a4ab8a81bedf32cb3588e57449967b8b4efe9f Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:54:14 -0700 Subject: [PATCH 018/128] test(sync): reproduce top-level pytest probe shadow --- tests/test_sync_core_runner.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index fc16d83e6..31c4e8fc5 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -258,6 +258,23 @@ def test_collection_probe_is_checker_owned_not_candidate_shadow(tmp_path) -> Non assert not (root / "candidate-probe-loaded").exists() +def test_collection_probe_fixed_name_is_not_candidate_shadowable(tmp_path) -> None: + root, head = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "pdd_checker_pytest_probe.py").write_text( + "import json, os\n" + "from pathlib import Path\n" + "def pytest_collection_modifyitems(items):\n" + " Path('candidate-fixed-probe-loaded').write_text('shadowed')\n" + " output = os.environ.get('PDD_TRUSTED_COLLECTION_OUTPUT')\n" + " if output:\n" + " Path(output).write_text(json.dumps([item.nodeid for item in items]))\n", + encoding="utf-8", + ) + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + assert not (root / "candidate-fixed-probe-loaded").exists() + + def test_deselected_declared_test_cannot_pass(tmp_path) -> None: content = "def test_keep(): assert True\ndef test_drop(): assert True\n" root, head = _repository(tmp_path, content) From a656364e16ff2de66cc9c821ef9c7a2f5d052ead Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:55:59 -0700 Subject: [PATCH 019/128] fix(sync): load pytest collection probe by absolute path --- pdd/sync_core/runner.py | 67 +++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 309f82340..0b7d6db5e 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -387,6 +387,49 @@ def _trusted_probe_plugin(directory: Path) -> tuple[str, Path]: return plugin_name, directory +def _checker_probe_digest() -> str: + """Return the digest of the checker-owned collection probe bytes.""" + return hashlib.sha256(_CHECKER_PYTEST_PROBE.read_bytes()).hexdigest() + + +def _trusted_collection_runner( + directory: Path, + root: Path, + pytest_args: list[str], +) -> Path: + """Create a checker-owned pytest entrypoint that imports the probe by path.""" + runner = directory / "run_collection.py" + runner.write_text( + "\n".join( + ( + "import importlib.util", + "import os", + "import sys", + "", + f"_ROOT = {json.dumps(str(root))}", + f"_PROBE = {json.dumps(str(_CHECKER_PYTEST_PROBE))}", + f"_ARGS = {json.dumps(pytest_args)}", + "", + "os.chdir(_ROOT)", + "_SPEC = importlib.util.spec_from_file_location(", + " '_pdd_checker_pytest_probe_abs', _PROBE", + ")", + "if _SPEC is None or _SPEC.loader is None:", + " raise ImportError('checker pytest probe is unavailable')", + "_MODULE = importlib.util.module_from_spec(_SPEC)", + "sys.modules['_pdd_checker_pytest_probe_abs'] = _MODULE", + "_SPEC.loader.exec_module(_MODULE)", + "", + "import pytest", + "raise SystemExit(pytest.main(_ARGS, plugins=[_MODULE]))", + "", + ) + ), + encoding="utf-8", + ) + return runner + + def _run_test_node( root: Path, node_id: str, @@ -440,26 +483,31 @@ def _collect_node_ids( with tempfile.TemporaryDirectory(prefix="pdd-trusted-collection-") as directory: temporary = Path(directory) collection_output = temporary / "node-ids.json" - plugin_name, plugin_directory = _trusted_probe_plugin(temporary) - command = [ - sys.executable, - "-m", - "pytest", + pytest_args = [ "--collect-only", "-q", "--strict-config", "--strict-markers", - "-p", - plugin_name, path.as_posix(), ] + runner = _trusted_collection_runner(temporary, root, pytest_args) + command = [sys.executable, str(runner)] digest = hashlib.sha256( - json.dumps(command, separators=(",", ":")).encode() + json.dumps( + { + "argv": [sys.executable, "pdd-trusted-collection-runner"], + "pytest_args": pytest_args, + "probe_path": str(_CHECKER_PYTEST_PROBE), + "probe_sha256": _checker_probe_digest(), + }, + separators=(",", ":"), + sort_keys=True, + ).encode() ).hexdigest() try: result = subprocess.run( command, - cwd=root, + cwd=temporary, capture_output=True, text=True, check=False, @@ -467,7 +515,6 @@ def _collect_node_ids( env=_pytest_environment() | { "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), - "PYTHONPATH": str(plugin_directory), }, ) except subprocess.TimeoutExpired: From e4f5eeba8a4e58ca31f3734703aa2baae7bc0795 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:55:59 -0700 Subject: [PATCH 020/128] test(heal): reproduce manual-merge operation failure --- tests/test_ci_drift_heal.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_ci_drift_heal.py b/tests/test_ci_drift_heal.py index 525bcbcf8..872bab8ce 100644 --- a/tests/test_ci_drift_heal.py +++ b/tests/test_ci_drift_heal.py @@ -4458,6 +4458,19 @@ def test_conflict_operation_returns_false(self): assert result is False mock_run.assert_not_called() + def test_manual_merge_terminal_operation_is_skipped_not_unknown(self): + """fail_and_request_manual_merge is a known terminal manual-conflict state.""" + drift = DriftInfo( + "auth", "python", "fail_and_request_manual_merge", + "Prompt and derived artifacts changed; manual conflict resolution required", + code_path="/repo/auth.py", + prompt_path="/repo/prompts/auth_python.prompt", + ) + with patch("pdd.ci_drift_heal.subprocess.run") as mock_run: + result = heal_module(drift, {"PDD_FORCE": "1"}) + assert result is None + mock_run.assert_not_called() + class TestCommitAndPushMessage: def test_multi_module_message_lists_all_basenames(self): From bab1c56f9f662846e3a2521fc81ada8202514ffa Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:56:23 -0700 Subject: [PATCH 021/128] fix(heal): skip manual-merge terminal operations --- pdd/ci_drift_heal.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pdd/ci_drift_heal.py b/pdd/ci_drift_heal.py index 4ff31130b..5d0913c25 100644 --- a/pdd/ci_drift_heal.py +++ b/pdd/ci_drift_heal.py @@ -1665,6 +1665,12 @@ def heal_module(drift: DriftInfo, env: Dict[str, str]) -> Optional[bool]: label=f"pdd sync {drift.basename}", ) return ok if ok else False + if op == "fail_and_request_manual_merge": + console.print( + f"[yellow]manual merge required for {drift.basename}; " + "skipping auto-heal[/yellow]" + ) + return None console.print(f"[red]unknown operation '{op}' for {drift.basename}[/red]") return False From c0bb303a048ee67bd8d104f0b3d47ba9ce6e1823 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 02:11:17 -0700 Subject: [PATCH 022/128] fix: harden candidate lifecycle wheel install --- pdd/commands/sync_core.py | 10 + pdd/sync_core/lifecycle.py | 82 ++++++++- tests/test_sync_core_lifecycle_scenarios.py | 192 +++++++++++++++++++- 3 files changed, 273 insertions(+), 11 deletions(-) diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py index 9a62750a7..3b95609fb 100644 --- a/pdd/commands/sync_core.py +++ b/pdd/commands/sync_core.py @@ -217,6 +217,16 @@ def certify( if "PDD_CANDIDATE_WHEEL" in os.environ else None ), + candidate_wheelhouse=( + Path(os.environ["PDD_CANDIDATE_WHEELHOUSE"]) + if "PDD_CANDIDATE_WHEELHOUSE" in os.environ + else None + ), + candidate_runtime_lock=( + Path(os.environ["PDD_CANDIDATE_RUNTIME_LOCK"]) + if "PDD_CANDIDATE_RUNTIME_LOCK" in os.environ + else None + ), cloud_root=targets[1].path, cloud_base_ref=targets[1].base_ref, cloud_head_ref=targets[1].head_ref, diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 2ff4d2ab1..6c800ac48 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -39,6 +39,17 @@ def _failed_result(*, timeout: bool = False) -> LifecycleResult: ) +def _required_paths_available( + path_inputs: tuple[tuple[Path | None, str], ...], +) -> bool: + """Return whether all required filesystem inputs satisfy their path predicate.""" + predicates = {"file": Path.is_file, "directory": Path.is_dir} + return all( + path is not None and predicates[predicate](Path(path)) + for path, predicate in path_inputs + ) + + def _normalized_results(payload: Any) -> dict[str, dict[str, Any]]: if not isinstance(payload, dict) or payload.get("schema_version") != 1: raise ValueError("released lifecycle result schema is invalid") @@ -104,10 +115,39 @@ def _dependency_environment_digest(candidate_python: Path, isolated: dict[str, s return hashlib.sha256(probe.stdout.encode()).hexdigest() +def _combined_candidate_lock(temporary: Path, runtime_lock: Path, wheel: Path) -> Path | None: + """Return a requirements file that pins the exact candidate wheel by hash.""" + try: + runtime_text = runtime_lock.read_text(encoding="utf-8") + wheel_hash = hashlib.sha256(wheel.read_bytes()).hexdigest() + except OSError: + return None + combined = temporary / "candidate-install.lock" + combined.write_text( + ( + runtime_text.rstrip() + + "\n" + + f"{wheel.resolve().as_posix()} --hash=sha256:{wheel_hash}\n" + ).lstrip(), + encoding="utf-8", + ) + return combined + + def _install_candidate_wheel( - temporary: Path, home: Path, wheel: Path + temporary: Path, + home: Path, + wheel: Path, + wheelhouse: Path, + runtime_lock: Path, ) -> tuple[Path, str] | None: - """Install the exact candidate wheel in a separate isolated environment.""" + # pylint: disable=too-many-return-statements + """Install the exact candidate wheel and runtime deps from a pinned wheelhouse.""" + if not wheelhouse.is_dir() or not runtime_lock.is_file(): + return None + combined_lock = _combined_candidate_lock(temporary, runtime_lock, wheel) + if combined_lock is None: + return None environment = temporary / "candidate-venv" isolated = _isolated_lifecycle_environment(home) created = subprocess.run( @@ -128,11 +168,15 @@ def _install_candidate_wheel( "-m", "pip", "install", - "--no-deps", + "--no-index", + "--find-links", + str(wheelhouse.resolve()), + "--require-hashes", "--only-binary=:all:", "--disable-pip-version-check", "--force-reinstall", - str(wheel.resolve()), + "-r", + str(combined_lock), ], capture_output=True, text=True, @@ -141,13 +185,27 @@ def _install_candidate_wheel( ) if installed.returncode != 0: return None - return candidate_python, _dependency_environment_digest(candidate_python, isolated) + proved = subprocess.run( + [str(candidate_python), "-I", "-m", "pdd.cli", "--help"], + capture_output=True, + text=True, + check=False, + env=isolated, + ) + if proved.returncode != 0: + return None + dependency_digest = _dependency_environment_digest(candidate_python, isolated) + if len(dependency_digest) != 64: + return None + return candidate_python, dependency_digest def run_lifecycle_matrix( root: Path, *, candidate_wheel: Path | None = None, + candidate_wheelhouse: Path | None = None, + candidate_runtime_lock: Path | None = None, cloud_root: Path | None = None, cloud_base_ref: str | None = None, cloud_head_ref: str | None = None, @@ -156,9 +214,13 @@ def run_lifecycle_matrix( # pylint: disable=too-many-arguments,too-many-locals """Run only the scenario harness installed with the released checker.""" del root # Candidate repository tests are never lifecycle evidence. + required_paths = ( + (candidate_wheel, "file"), + (candidate_wheelhouse, "directory"), + (candidate_runtime_lock, "file"), + ) if ( - candidate_wheel is None - or not Path(candidate_wheel).is_file() + not _required_paths_available(required_paths) or cloud_root is None or cloud_base_ref is None or cloud_head_ref is None @@ -169,7 +231,11 @@ def run_lifecycle_matrix( output = temporary / "result.json" (temporary / "home").mkdir(mode=0o700) installed_candidate = _install_candidate_wheel( - temporary, temporary / "home", Path(candidate_wheel) + temporary, + temporary / "home", + Path(candidate_wheel), + Path(candidate_wheelhouse), + Path(candidate_runtime_lock), ) if installed_candidate is None: return _failed_result() diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index b4333558a..abd1b4937 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -1,16 +1,23 @@ """Process-level merge, wheel, and real-consumer lifecycle scenarios.""" +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name import argparse +import hashlib import os import json import subprocess import sys +import zipfile from pathlib import Path, PurePosixPath import pytest from pdd.sync_core.certificate import LifecycleResult -from pdd.sync_core.lifecycle import _isolated_lifecycle_environment +from pdd.sync_core.lifecycle import ( + _install_candidate_wheel, + _isolated_lifecycle_environment, + run_lifecycle_matrix, +) from pdd.sync_core.scenario_contract import REQUIRED_SCENARIOS from pdd.sync_core import scenario_harness from pdd.sync_core import ( @@ -44,9 +51,40 @@ def _record_metric(name: str, value: int) -> None: if not configured: return path = Path(configured) - payload = json.loads(path.read_text()) if path.exists() else {} + payload = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} payload[name] = value - path.write_text(json.dumps(payload, sort_keys=True)) + path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + + +def _write_wheel( + wheelhouse: Path, + *, + distribution: str, + version: str, + files: dict[str, str], + metadata_extra: str = "", +) -> Path: + dist_info = f"{distribution.replace('-', '_')}-{version}.dist-info" + wheel = wheelhouse / f"{distribution.replace('-', '_')}-{version}-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for name, content in files.items(): + archive.writestr(name, content) + archive.writestr( + f"{dist_info}/METADATA", + ( + "Metadata-Version: 2.1\n" + f"Name: {distribution}\n" + f"Version: {version}\n" + f"{metadata_extra}\n" + ), + ) + archive.writestr( + f"{dist_info}/WHEEL", + "Wheel-Version: 1.0\nGenerator: pdd-test\nRoot-Is-Purelib: true\n" + "Tag: py3-none-any\n", + ) + archive.writestr(f"{dist_info}/RECORD", "") + return wheel def test_lifecycle_contract_requires_public_repair_injection_scenarios() -> None: @@ -74,6 +112,154 @@ def test_lifecycle_predicate_requires_dependency_environment_digest() -> None: assert result.dependency_environment_digest == "b" * 64 +def test_lifecycle_matrix_fails_closed_without_hash_pinned_wheelhouse( + tmp_path, +) -> None: + wheel = tmp_path / "pdd_cli-1.0.0-py3-none-any.whl" + wheel.write_bytes(b"candidate-wheel") + result = run_lifecycle_matrix( + tmp_path, + candidate_wheel=wheel, + cloud_root=tmp_path, + cloud_base_ref="a" * 40, + cloud_head_ref="b" * 40, + ) + assert result.failed == len(REQUIRED_SCENARIOS) + assert result.candidate_wheel_sha256 == "" + assert result.dependency_environment_digest == "" + + +def test_candidate_install_uses_hash_pinned_wheelhouse_no_index( + tmp_path, monkeypatch +) -> None: + wheel = tmp_path / "pdd_cli-1.0.0-py3-none-any.whl" + wheel.write_bytes(b"candidate-wheel") + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + lock = tmp_path / "runtime.lock" + lock.write_text( + "click==8.4.1 --hash=sha256:" + ("c" * 64) + "\n", + encoding="utf-8", + ) + calls = [] + + def fake_run(command, **kwargs): + calls.append((tuple(str(item) for item in command), kwargs)) + if "-m" in command and "venv" in command: + (tmp_path / "candidate-venv" / ("Scripts" if os.name == "nt" else "bin")).mkdir( + parents=True + ) + return subprocess.CompletedProcess(command, 0, "ok", "") + + monkeypatch.setattr("pdd.sync_core.lifecycle.subprocess.run", fake_run) + installed = _install_candidate_wheel( + tmp_path, + tmp_path / "home", + wheel, + wheelhouse, + lock, + ) + assert installed is not None + install_command = calls[1][0] + assert "--no-index" in install_command + assert "--require-hashes" in install_command + assert "--find-links" in install_command + assert str(wheelhouse.resolve()) in install_command + assert "--no-deps" not in install_command + combined_lock = Path(install_command[install_command.index("-r") + 1]) + lock_text = combined_lock.read_text(encoding="utf-8") + assert str(wheel.resolve()) in lock_text + assert f"--hash=sha256:{hashlib.sha256(wheel.read_bytes()).hexdigest()}" in lock_text + + +def test_candidate_install_proves_isolated_module_entrypoint( + tmp_path, monkeypatch +) -> None: + wheel = tmp_path / "pdd_cli-1.0.0-py3-none-any.whl" + wheel.write_bytes(b"candidate-wheel") + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + lock = tmp_path / "runtime.lock" + lock.write_text("", encoding="utf-8") + calls = [] + + def fake_run(command, **_kwargs): + calls.append(tuple(str(item) for item in command)) + if "-m" in command and "venv" in command: + (tmp_path / "candidate-venv" / ("Scripts" if os.name == "nt" else "bin")).mkdir( + parents=True + ) + return subprocess.CompletedProcess(command, 0, "ok", "") + + monkeypatch.setattr("pdd.sync_core.lifecycle.subprocess.run", fake_run) + assert _install_candidate_wheel( + tmp_path, + tmp_path / "home", + wheel, + wheelhouse, + lock, + ) + assert any( + command[-4:] == ("-I", "-m", "pdd.cli", "--help") + for command in calls + ) + + +def test_candidate_install_e2e_uses_locked_runtime_wheelhouse(tmp_path) -> None: + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + runtime = _write_wheel( + wheelhouse, + distribution="runtime-dep", + version="1.0.0", + files={"runtime_dep.py": "VALUE = 'runtime wheel loaded'\n"}, + ) + candidate = _write_wheel( + tmp_path, + distribution="pdd-cli", + version="1.0.0", + files={ + "pdd/__init__.py": "", + "pdd/cli.py": ( + "import sys\n" + "import runtime_dep\n" + "if __name__ == '__main__':\n" + " print(runtime_dep.VALUE)\n" + " raise SystemExit(0 if '--help' in sys.argv else 2)\n" + ), + }, + metadata_extra="Requires-Dist: runtime-dep==1.0.0\n", + ) + lock = tmp_path / "runtime.lock" + lock.write_text( + "runtime-dep==1.0.0 --hash=sha256:" + f"{hashlib.sha256(runtime.read_bytes()).hexdigest()}\n", + encoding="utf-8", + ) + installed = _install_candidate_wheel( + tmp_path, + tmp_path / "home", + candidate, + wheelhouse, + lock, + ) + assert installed is not None + candidate_python, dependency_digest = installed + pyvenv = candidate_python.parents[1] / "pyvenv.cfg" + assert "include-system-site-packages = false" in pyvenv.read_text(encoding="utf-8") + probe = _run( + tmp_path, + str(candidate_python), + "-I", + "-m", + "pdd.cli", + "--help", + ) + assert probe.returncode == 0 + assert "runtime wheel loaded" in probe.stdout + assert len(dependency_digest) == 64 + + def test_lifecycle_environment_strips_signer_capabilities(tmp_path, monkeypatch) -> None: monkeypatch.setenv("PDD_ATTESTATION_SIGNER_COMMAND", "protected-attestation") monkeypatch.setenv("PDD_CERTIFICATE_ISSUER", "protected-issuer") From 6bf14c6559d6fea5612b5743f8d9962b84c026ee Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 04:57:25 -0700 Subject: [PATCH 023/128] test(sync): cover pytest support isolation gaps --- tests/test_sync_core_runner.py | 118 +++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 31c4e8fc5..5f8c5c00c 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -147,6 +147,65 @@ def test_candidate_modified_imported_test_helper_cannot_self_certify(tmp_path) - assert "tests/helper.py" in executions[0].detail +def test_candidate_modified_product_code_can_be_certified_by_protected_tests(tmp_path) -> None: + root, base = _repository( + tmp_path, + "import product\n\ndef test_widget(): assert product.expected() >= 1\n", + ) + (root / "product.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "base product") + base = _git(root, "rev-parse", "HEAD") + (root / "product.py").write_text("def expected(): return 2\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate product") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.PASS + + +def test_string_pytest_plugins_are_bound_as_protected_support(tmp_path) -> None: + root, _initial = _repository( + tmp_path, + "def test_widget(value): assert value in {1, 2}\n", + ) + (root / "tests/__init__.py").write_text("") + (root / "tests/plugin.py").write_text( + "import pytest\n@pytest.fixture\ndef value(): return 1\n" + ) + (root / "conftest.py").write_text('pytest_plugins = "tests.plugin"\n') + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "base string plugin") + base = _git(root, "rev-parse", "HEAD") + (root / "tests/plugin.py").write_text( + "import pytest\n@pytest.fixture\ndef value(): return 2\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate plugin change") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "tests/plugin.py" in executions[0].detail + + +def test_dynamic_pytest_plugins_fail_closed(tmp_path) -> None: + root, _initial = _repository( + tmp_path, + "def test_widget(value): assert value == 1\n", + ) + (root / "tests/__init__.py").write_text("") + (root / "tests/plugin.py").write_text( + "import pytest\n@pytest.fixture\ndef value(): return 1\n" + ) + (root / "conftest.py").write_text("pytest_plugins = tuple(['tests.plugin'])\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "dynamic plugin declaration") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "dynamic pytest_plugins" in executions[0].detail + + def test_candidate_modified_importfrom_alias_helper_cannot_self_certify(tmp_path) -> None: root, _initial = _repository( tmp_path, @@ -204,6 +263,65 @@ def test_validator_subprocess_cannot_read_signing_secret(tmp_path, monkeypatch) assert (root / "observed-secret").read_text() == "missing" +def test_pytest_execution_uses_private_home_and_userprofile(tmp_path, monkeypatch) -> None: + protected_home = tmp_path / "protected-home" + protected_home.mkdir() + (protected_home / "secret.txt").write_text("protected-secret") + monkeypatch.setenv("HOME", str(protected_home)) + monkeypatch.setenv("USERPROFILE", str(protected_home)) + content = ( + "import json, os\nfrom pathlib import Path\n" + "def test_widget():\n" + " home = Path(os.environ['HOME'])\n" + " profile = Path(os.environ['USERPROFILE'])\n" + " Path('observed-execution-home.json').write_text(json.dumps({\n" + " 'home': str(home),\n" + " 'userprofile': str(profile),\n" + " 'home_secret': (home / 'secret.txt').exists(),\n" + " 'profile_secret': (profile / 'secret.txt').exists(),\n" + " }, sort_keys=True))\n" + ) + root, head = _repository(tmp_path, content) + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + observed = json.loads((root / "observed-execution-home.json").read_text()) + assert observed["home"] != str(protected_home) + assert observed["userprofile"] != str(protected_home) + assert observed["home_secret"] is False + assert observed["profile_secret"] is False + + +def test_pytest_collection_uses_private_home_and_userprofile(tmp_path, monkeypatch) -> None: + protected_home = tmp_path / "protected-home" + protected_home.mkdir() + (protected_home / "secret.txt").write_text("protected-secret") + monkeypatch.setenv("HOME", str(protected_home)) + monkeypatch.setenv("USERPROFILE", str(protected_home)) + root, head = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "conftest.py").write_text( + "import json, os\nfrom pathlib import Path\n" + "def pytest_collection_modifyitems(items):\n" + " home = Path(os.environ['HOME'])\n" + " profile = Path(os.environ['USERPROFILE'])\n" + " Path('observed-collection-home.json').write_text(json.dumps({\n" + " 'home': str(home),\n" + " 'userprofile': str(profile),\n" + " 'home_secret': (home / 'secret.txt').exists(),\n" + " 'profile_secret': (profile / 'secret.txt').exists(),\n" + " }, sort_keys=True))\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "collection hook") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + observed = json.loads((root / "observed-collection-home.json").read_text()) + assert observed["home"] != str(protected_home) + assert observed["userprofile"] != str(protected_home) + assert observed["home_secret"] is False + assert observed["profile_secret"] is False + + def test_validator_subprocess_cannot_read_signer_capabilities(tmp_path, monkeypatch) -> None: content = ( "import json, os\nfrom pathlib import Path\n" From f58655122761e46f891828600d3aedc51ec8d7e6 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 04:57:25 -0700 Subject: [PATCH 024/128] fix(sync): bind pytest support and private home --- pdd/sync_core/isolation.py | 1 + pdd/sync_core/runner.py | 162 ++++++++++++++++++++++++++++++++----- 2 files changed, 144 insertions(+), 19 deletions(-) diff --git a/pdd/sync_core/isolation.py b/pdd/sync_core/isolation.py index 0ea7cd3ce..5fbce14d4 100644 --- a/pdd/sync_core/isolation.py +++ b/pdd/sync_core/isolation.py @@ -57,6 +57,7 @@ def untrusted_child_environment( } if home is not None: environment["HOME"] = str(home) + environment["USERPROFILE"] = str(home) environment["XDG_CONFIG_HOME"] = str(home / ".config") environment["XDG_CACHE_HOME"] = str(home / ".cache") for key in drop or set(): diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 0b7d6db5e..c41235482 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -78,16 +78,64 @@ class AttestationIssue: issued_at: datetime +def _pytest_plugin_modules(node: ast.AST) -> tuple[set[str], bool]: + """Return statically declared pytest plugin modules and dynamic status.""" + modules: set[str] = set() + if isinstance(node, ast.Constant) and isinstance(node.value, str): + modules.add(node.value) + return modules, False + if isinstance(node, (ast.List, ast.Tuple)): + for item in node.elts: + if not isinstance(item, ast.Constant) or not isinstance(item.value, str): + return modules, True + modules.add(item.value) + return modules, False + return modules, True + + +def _pytest_plugin_declaration_targets(node: ast.AST) -> tuple[ast.AST, ...]: + """Return assignment targets that may declare pytest plugins.""" + if isinstance(node, ast.Assign): + return tuple(node.targets) + if isinstance(node, ast.AnnAssign): + return (node.target,) + return () + + +def _declares_pytest_plugins(targets: tuple[ast.AST, ...]) -> bool: + """Return whether assignment targets include the pytest_plugins sentinel.""" + return any( + isinstance(target, ast.Name) and target.id == "pytest_plugins" + for target in targets + ) + + +def _is_pytest_support_path(path: PurePosixPath) -> bool: + """Return whether a path is pytest support rather than product code.""" + return ( + path.name == "conftest.py" + or path in PYTEST_CONFIG_PATHS + or "tests" in path.parts + or path.name.startswith("test_") + ) + + def _local_module_paths( - root: Path, ref: str, source_path: PurePosixPath, source: bytes -) -> set[PurePosixPath]: + root: Path, + ref: str, + source_path: PurePosixPath, + source: bytes, + *, + support_only: bool = False, +) -> tuple[set[PurePosixPath], bool]: # pylint: disable=too-many-locals """Resolve repository-local Python imports without executing candidate code.""" try: tree = ast.parse(source) except (SyntaxError, UnicodeDecodeError): - return set() + return set(), False modules: set[str] = set() + dynamic_pytest_plugins = False for node in ast.walk(tree): if isinstance(node, ast.Import): modules.update(alias.name for alias in node.names) @@ -101,15 +149,10 @@ def _local_module_paths( continue separator = "" if not module or module.endswith(".") else "." modules.add(f"{module}{separator}{alias.name}") - elif isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == "pytest_plugins" - for target in node.targets - ): - values = node.value.elts if isinstance(node.value, (ast.List, ast.Tuple)) else () - modules.update( - item.value for item in values if isinstance(item, ast.Constant) - and isinstance(item.value, str) - ) + elif _declares_pytest_plugins(_pytest_plugin_declaration_targets(node)): + declared, dynamic = _pytest_plugin_modules(node.value) + modules.update(declared) + dynamic_pytest_plugins = dynamic_pytest_plugins or dynamic resolved: set[PurePosixPath] = set() for module in modules: level = len(module) - len(module.lstrip(".")) @@ -125,9 +168,12 @@ def _local_module_paths( module_path / "__init__.py", ) resolved.update( - path for path in candidates if read_git_blob(root, ref, path) is not None + path + for path in candidates + if read_git_blob(root, ref, path) is not None + and (not support_only or _is_pytest_support_path(path)) ) - return resolved + return resolved, dynamic_pytest_plugins def _pytest_support_closure( @@ -140,6 +186,7 @@ def _pytest_support_closure( ref, pending=tuple(test_paths) + tuple(config_paths), included=config_paths, + test_artifacts=test_paths, ) @@ -170,10 +217,12 @@ def _transitive_support_blobs( *, pending: tuple[PurePosixPath, ...], included: set[PurePosixPath], + test_artifacts: tuple[PurePosixPath, ...] = (), ) -> tuple[tuple[PurePosixPath, bytes], ...]: """Resolve local Python imports from protected runner support paths.""" paths = set(included) remaining = list(pending) + test_artifact_set = set(test_artifacts) visited: set[PurePosixPath] = set() while remaining: path = remaining.pop() @@ -183,7 +232,14 @@ def _transitive_support_blobs( source = read_git_blob(root, ref, path) if source is None or path.suffix != ".py": continue - discovered = _local_module_paths(root, ref, path, source) - visited + discovered, _dynamic = _local_module_paths( + root, + ref, + path, + source, + support_only=path in test_artifact_set, + ) + discovered -= visited paths.update(discovered) remaining.extend(discovered) blobs = ((path, read_git_blob(root, ref, path)) for path in sorted(paths)) @@ -207,6 +263,35 @@ def pytest_validator_config_digest( return digest.hexdigest() +def _has_dynamic_pytest_plugins( + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] +) -> bool: + """Fail closed when pytest plugin declarations cannot be statically bound.""" + config_paths = _pytest_config_paths(root, ref, test_paths) + remaining = list(tuple(test_paths) + tuple(config_paths)) + test_artifact_set = set(test_paths) + visited: set[PurePosixPath] = set() + while remaining: + path = remaining.pop() + if path in visited: + continue + visited.add(path) + source = read_git_blob(root, ref, path) + if source is None or path.suffix != ".py": + continue + discovered, dynamic = _local_module_paths( + root, + ref, + path, + source, + support_only=path in test_artifact_set, + ) + if dynamic: + return True + remaining.extend(discovered - visited) + return False + + def _support_digest( root: Path, ref: str, profile: VerificationProfile ) -> tuple[str, tuple[PurePosixPath, ...]]: @@ -355,9 +440,10 @@ def _junit_outcome( return outcome, detail -def _pytest_environment() -> dict[str, str]: +def _pytest_environment(home: Path) -> dict[str, str]: """Return the protected credential-free pytest process environment.""" return untrusted_child_environment( + home=home, extra={"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PYTHONNOUSERSITE": "1"}, drop={"PYTEST_ADDOPTS", "PYTHONPATH"}, ) @@ -421,6 +507,8 @@ def _trusted_collection_runner( "_SPEC.loader.exec_module(_MODULE)", "", "import pytest", + "if _ROOT not in sys.path:", + " sys.path.insert(0, _ROOT)", "raise SystemExit(pytest.main(_ARGS, plugins=[_MODULE]))", "", ) @@ -447,7 +535,10 @@ def _run_test_node( json.dumps(command, separators=(",", ":")).encode() ).hexdigest() with tempfile.TemporaryDirectory(prefix="pdd-trusted-runner-") as directory: - junit = Path(directory) / "junit.xml" + temporary = Path(directory) + home = temporary / "home" + home.mkdir(mode=0o700) + junit = temporary / "junit.xml" try: result = subprocess.run( [*command, f"--junitxml={junit}"], @@ -456,7 +547,7 @@ def _run_test_node( text=True, check=False, timeout=timeout_seconds, - env=_pytest_environment(), + env=_pytest_environment(home), ) except subprocess.TimeoutExpired: return RunnerExecution( @@ -479,9 +570,12 @@ def _collect_node_ids( path: PurePosixPath, timeout_seconds: int, ) -> tuple[RunnerExecution, tuple[str, ...]]: + # pylint: disable=too-many-locals """Collect exact pytest node IDs through the protected adapter.""" with tempfile.TemporaryDirectory(prefix="pdd-trusted-collection-") as directory: temporary = Path(directory) + home = temporary / "home" + home.mkdir(mode=0o700) collection_output = temporary / "node-ids.json" pytest_args = [ "--collect-only", @@ -512,7 +606,7 @@ def _collect_node_ids( text=True, check=False, timeout=timeout_seconds, - env=_pytest_environment() + env=_pytest_environment(home) | { "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), }, @@ -654,6 +748,7 @@ def _obligation_preflight( base_sha: str, head_sha: str, ) -> RunnerExecution | None: + # pylint: disable=too-many-return-statements """Return a normalized fail-closed result before executing pytest.""" if obligation.kind.casefold() != "test" or obligation.validator_id != "pytest": return RunnerExecution( @@ -674,12 +769,41 @@ def _obligation_preflight( for ref in (base_sha, head_sha) } if expected_config_digests != {obligation.validator_config_digest}: + support_paths = tuple( + path + for path, _content in _pytest_support_closure( + root, head_sha, obligation.artifact_paths + ) + ) + changed_support = _changed_paths(root, base_sha, head_sha, support_paths) + direct_config = { + path.as_posix() + for path in _pytest_config_paths(root, head_sha, obligation.artifact_paths) + } + plugin_support = sorted(set(changed_support) - direct_config) + if plugin_support: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.QUARANTINED, + obligation.validator_config_digest, + "candidate-modified pytest support cannot solely certify itself: " + + ", ".join(plugin_support), + ) return RunnerExecution( obligation.obligation_id, EvidenceOutcome.ERROR, obligation.validator_config_digest, "declared validator config digest does not match protected closure", ) + if _has_dynamic_pytest_plugins(root, base_sha, obligation.artifact_paths) or ( + _has_dynamic_pytest_plugins(root, head_sha, obligation.artifact_paths) + ): + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "dynamic pytest_plugins declarations are not bound by this adapter", + ) if _config_loads_plugin(root, base_sha) or _config_loads_plugin(root, head_sha): return RunnerExecution( obligation.obligation_id, From 24f21d861ceef5c473421a7fd05d42dbeeebd3b0 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 18:25:17 -0700 Subject: [PATCH 025/128] test(sync): reproduce unbound candidate wheel provenance --- ...sync_core_candidate_artifact_provenance.py | 153 ++++++++++++++++++ tests/test_sync_core_runner.py | 74 ++++++++- 2 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 tests/test_sync_core_candidate_artifact_provenance.py diff --git a/tests/test_sync_core_candidate_artifact_provenance.py b/tests/test_sync_core_candidate_artifact_provenance.py new file mode 100644 index 000000000..845d07d97 --- /dev/null +++ b/tests/test_sync_core_candidate_artifact_provenance.py @@ -0,0 +1,153 @@ +"""Strict tests for protected candidate-wheel build provenance.""" + +import hashlib +import json +from datetime import datetime, timedelta, timezone + +import pytest + +from pdd.sync_core import AttestationSigner +from pdd.sync_core.artifact_provenance import ( + CandidateArtifactPolicy, + CandidateArtifactProvenanceError, + load_candidate_artifact_provenance, +) + + +SOURCE_SHA = "a" * 40 +LOCK_SHA256 = "b" * 64 +WORKFLOW = "promptdriven/pdd/.github/workflows/candidate-wheel.yml@refs/heads/main" +TARGET = { + "implementation": "CPython", + "version": "3.12.3", + "abi": "cp312", + "platform": "manylinux_2_17_x86_64", +} + + +def _policy(authority: AttestationSigner) -> CandidateArtifactPolicy: + return CandidateArtifactPolicy( + authority.issuer, + authority.public_key_bytes(), + WORKFLOW, + LOCK_SHA256, + TARGET["implementation"], + TARGET["version"], + TARGET["abi"], + TARGET["platform"], + ) + + +def _attestation( + wheel, + authority: AttestationSigner, + **overrides, +) -> dict: + now = datetime.now(timezone.utc) + payload = { + "schema_version": 1, + "issuer": authority.issuer, + "attestation_id": "candidate-build-123", + "nonce": "candidate-build-nonce-123", + "issued_at": now.isoformat(), + "expires_at": (now + timedelta(minutes=10)).isoformat(), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "source_sha": SOURCE_SHA, + "dependency_lock_sha256": LOCK_SHA256, + "python": TARGET, + "builder_workflow_identity": WORKFLOW, + } + payload.update(overrides) + payload["signature"] = { + "algorithm": "Ed25519", + "value": authority.sign_bytes( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + ), + } + return payload + + +def _load(tmp_path, wheel, authority, **overrides): + path = tmp_path / "candidate-build-attestation.json" + path.write_text(json.dumps(_attestation(wheel, authority, **overrides))) + return load_candidate_artifact_provenance(path, wheel, _policy(authority)) + + +def test_unrelated_wheel_cannot_use_another_wheels_attestation(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + certified = tmp_path / "certified.whl" + unrelated = tmp_path / "unrelated.whl" + certified.write_bytes(b"certified candidate wheel") + unrelated.write_bytes(b"unrelated candidate wheel") + with pytest.raises(CandidateArtifactProvenanceError, match="digest"): + _load(tmp_path, unrelated, authority, wheel_sha256=hashlib.sha256(certified.read_bytes()).hexdigest()) + + +def test_correct_wheel_with_wrong_certified_source_sha_is_rejected(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + with pytest.raises(CandidateArtifactProvenanceError, match="source SHA"): + _load(tmp_path, wheel, authority, source_sha="c" * 40) + + +def test_forged_build_attestation_is_rejected(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + path = tmp_path / "candidate-build-attestation.json" + forged = _attestation(wheel, authority) + forged["source_sha"] = "c" * 40 + path.write_text(json.dumps(forged)) + with pytest.raises(CandidateArtifactProvenanceError, match="signature"): + load_candidate_artifact_provenance(path, wheel, _policy(authority)) + + +def test_stale_or_replayed_build_attestation_is_rejected(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + with pytest.raises(CandidateArtifactProvenanceError, match="expired"): + _load( + tmp_path, + wheel, + authority, + issued_at=(datetime.now(timezone.utc) - timedelta(hours=2)).isoformat(), + expires_at=(datetime.now(timezone.utc) - timedelta(hours=1)).isoformat(), + ) + provenance = _load(tmp_path, wheel, authority) + with pytest.raises(CandidateArtifactProvenanceError, match="replayed"): + provenance.verify(_policy(authority), expected_source_sha=SOURCE_SHA) + + +@pytest.mark.parametrize( + ("field", "value", "error"), + [ + ("builder_workflow_identity", "attacker/workflow@main", "builder workflow"), + ("dependency_lock_sha256", "c" * 64, "dependency lock"), + ("python", {**TARGET, "abi": "cp313"}, "interpreter"), + ], +) +def test_wrong_protected_build_environment_is_rejected(tmp_path, field, value, error) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + with pytest.raises(CandidateArtifactProvenanceError, match=error): + _load(tmp_path, wheel, authority, **{field: value}) + + +def test_source_checkout_is_not_a_candidate_artifact(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + checkout = tmp_path / "pdd" + checkout.mkdir() + with pytest.raises(CandidateArtifactProvenanceError, match="wheel path"): + _load(tmp_path, checkout, authority) + + +def test_valid_exact_sha_artifact_is_accepted(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + provenance = _load(tmp_path, wheel, authority) + assert provenance.source_sha == SOURCE_SHA + assert provenance.wheel_sha256 == hashlib.sha256(wheel.read_bytes()).hexdigest() diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 5f8c5c00c..4b2237273 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -41,7 +41,11 @@ def _repository(tmp_path: Path, test_content: str) -> tuple[Path, str]: return root, _git(root, "rev-parse", "HEAD") -def _profile(root: Path, ref: str) -> VerificationProfile: +def _profile( + root: Path, + ref: str, + code_under_test_paths: tuple[PurePosixPath, ...] = (), +) -> VerificationProfile: paths = (PurePosixPath("tests/test_widget.py"),) obligation = VerificationObligation( "pytest", @@ -50,14 +54,20 @@ def _profile(root: Path, ref: str) -> VerificationProfile: pytest_validator_config_digest(root, ref, paths), ("REQ-1",), paths, + code_under_test_paths=code_under_test_paths, ) return VerificationProfile(UNIT, (obligation,), ("REQ-1",), "profile-v1") -def _run(root: Path, base: str, head: str): +def _run( + root: Path, + base: str, + head: str, + code_under_test_paths: tuple[PurePosixPath, ...] = (), +): return run_profile( root, - _profile(root, base), + _profile(root, base, code_under_test_paths), RunBinding("snapshot-v1", base, head), AttestationIssue( AttestationSigner("trusted-ci", b"v" * 32), @@ -160,10 +170,66 @@ def test_candidate_modified_product_code_can_be_certified_by_protected_tests(tmp _git(root, "add", ".") _git(root, "commit", "-q", "-m", "candidate product") head = _git(root, "rev-parse", "HEAD") - _envelope, executions = _run(root, base, head) + _envelope, executions = _run( + root, base, head, (PurePosixPath("product.py"),) + ) assert executions[0].outcome is EvidenceOutcome.PASS +def test_candidate_modified_helper_outside_tests_is_protected(tmp_path) -> None: + root, _initial = _repository( + tmp_path, + "from support.fixtures import expected\n" + "def test_widget(): assert expected() == 1\n", + ) + (root / "support").mkdir() + (root / "support/__init__.py").write_text("") + (root / "support/fixtures.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "base helper") + base = _git(root, "rev-parse", "HEAD") + (root / "support/fixtures.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "support/fixtures.py" in executions[0].detail + + +def test_collection_time_protected_test_rewrite_fails_closed(tmp_path) -> None: + root, _initial = _repository( + tmp_path, + "import product\ndef test_widget(): assert False\n", + ) + (root / "product.py").write_text( + "from pathlib import Path\n" + "Path('tests/test_widget.py').write_text(" + "'import product\\ndef test_widget(): assert True\\n')\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate product") + head = _git(root, "rev-parse", "HEAD") + original = (root / "tests/test_widget.py").read_bytes() + _envelope, executions = _run( + root, head, head, (PurePosixPath("product.py"),) + ) + assert executions[0].outcome is not EvidenceOutcome.PASS + assert "protected" in executions[0].detail + assert (root / "tests/test_widget.py").read_bytes() == original + + +def test_external_literal_pytest_plugin_fails_closed(tmp_path) -> None: + root, _initial = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "conftest.py").write_text('pytest_plugins = "pytest_mock"\n') + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "external plugin") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "external pytest_plugins" in executions[0].detail + + def test_string_pytest_plugins_are_bound_as_protected_support(tmp_path) -> None: root, _initial = _repository( tmp_path, From 6e8f955fd51cbb68998cd95dae005d9b6efc5280 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 18:33:17 -0700 Subject: [PATCH 026/128] fix(sync): bind candidate wheel to certified source --- pdd/commands/sync_core.py | 9 + pdd/sync_core/__init__.py | 12 + pdd/sync_core/artifact_provenance.py | 290 ++++++++++++++++++ pdd/sync_core/certificate.py | 98 +++++- pdd/sync_core/lifecycle.py | 19 +- ...sync_core_candidate_artifact_provenance.py | 20 +- tests/test_sync_core_certificate.py | 96 +++++- 7 files changed, 531 insertions(+), 13 deletions(-) create mode 100644 pdd/sync_core/artifact_provenance.py diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py index 3b95609fb..ae0d77228 100644 --- a/pdd/commands/sync_core.py +++ b/pdd/commands/sync_core.py @@ -28,6 +28,7 @@ build_global_certificate, build_unit_manifest, build_unit_snapshot, + candidate_artifact_policy_from_environment, checker_identity_from_environment, encode_fingerprint, finalize_unit, @@ -206,6 +207,7 @@ def certify( if replay_ledger.exists() and not replay_ledger.is_dir(): raise ValueError("global --replay-ledger must be a directory") replay_ledger.mkdir(parents=True, exist_ok=True) + candidate_policy = candidate_artifact_policy_from_environment() report = build_global_certificate( targets, GlobalCertificateOptions( @@ -227,6 +229,12 @@ def certify( if "PDD_CANDIDATE_RUNTIME_LOCK" in os.environ else None ), + candidate_attestation=( + Path(os.environ["PDD_CANDIDATE_BUILD_ATTESTATION"]) + if "PDD_CANDIDATE_BUILD_ATTESTATION" in os.environ + else None + ), + candidate_artifact_policy=candidate_policy, cloud_root=targets[1].path, cloud_base_ref=targets[1].base_ref, cloud_head_ref=targets[1].head_ref, @@ -238,6 +246,7 @@ def certify( ), required_nightly_streak=int(options["require_nightly_streak"]), checker_identity=checker_identity_from_environment(), + candidate_artifact_policy=candidate_policy, nightly_observation=_load_nightly_observation( options.get("nightly_observation") ), diff --git a/pdd/sync_core/__init__.py b/pdd/sync_core/__init__.py index 3e3f12a96..e16779e6e 100644 --- a/pdd/sync_core/__init__.py +++ b/pdd/sync_core/__init__.py @@ -15,6 +15,13 @@ signer_from_environment, verify_global_certificate, ) +from .artifact_provenance import ( + CandidateArtifactPolicy, + CandidateArtifactProvenance, + CandidateArtifactProvenanceError, + candidate_artifact_policy_from_environment, + load_candidate_artifact_provenance, +) from .identity import ( RepositoryIdentity, RepositoryIdentityError, @@ -140,6 +147,9 @@ "AttestationTrustPolicy", "BaselineStatus", "CandidateId", + "CandidateArtifactPolicy", + "CandidateArtifactProvenance", + "CandidateArtifactProvenanceError", "CandidateRecord", "CanonicalCounts", "CanonicalFinalizationError", @@ -209,6 +219,7 @@ "WaiverSet", "classify", "canonical_repository_id", + "candidate_artifact_policy_from_environment", "canonical_root_for_paths", "decode_attestation", "encode_attestation", @@ -228,6 +239,7 @@ "include_paths", "load_verification_profiles", "load_attestation", + "load_candidate_artifact_provenance", "load_trust_policy", "load_sync_waivers", "initialize_repository_identity", diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py new file mode 100644 index 000000000..d29dab56c --- /dev/null +++ b/pdd/sync_core/artifact_provenance.py @@ -0,0 +1,290 @@ +"""Verification of protected build attestations for candidate wheels.""" +# pylint: disable=duplicate-code,too-many-instance-attributes,too-many-boolean-expressions + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + + +class CandidateArtifactProvenanceError(ValueError): + """Raised when a candidate wheel lacks protected build provenance.""" + + +def _canonical_bytes(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _timestamp(value: object, field_name: str) -> datetime: + try: + parsed = datetime.fromisoformat(str(value)) + except ValueError as exc: + raise CandidateArtifactProvenanceError( + f"candidate attestation {field_name} is invalid" + ) from exc + if parsed.tzinfo is None: + raise CandidateArtifactProvenanceError( + f"candidate attestation {field_name} is not timezone-aware" + ) + return parsed.astimezone(timezone.utc) + + +def _sha256_value(value: object, field_name: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise CandidateArtifactProvenanceError( + f"candidate attestation {field_name} is invalid" + ) + return value + + +@dataclass +class CandidateArtifactPolicy: + """Protected expected identity of a candidate-wheel build lane.""" + + issuer: str + public_key: bytes + builder_workflow_identity: str + dependency_lock_sha256: str + python_implementation: str + python_version: str + python_abi: str + python_platform: str + _consumed: set[tuple[str, str, str]] = field(default_factory=set, init=False) + + def __post_init__(self) -> None: + if ( + not self.issuer + or len(self.public_key) != 32 + or not self.builder_workflow_identity + or not self.python_implementation + or not self.python_version + or not self.python_abi + or not self.python_platform + ): + raise ValueError("candidate artifact policy is malformed") + _sha256_value(self.dependency_lock_sha256, "dependency lock digest") + + def identity(self) -> dict[str, str]: + """Return public policy claims that the certificate must bind.""" + return { + "issuer": self.issuer, + "builder_workflow_identity": self.builder_workflow_identity, + "dependency_lock_sha256": self.dependency_lock_sha256, + "python_implementation": self.python_implementation, + "python_version": self.python_version, + "python_abi": self.python_abi, + "python_platform": self.python_platform, + } + + def consume(self, attestation_id: str, nonce: str) -> None: + """Reject a build statement reused by a second certification attempt.""" + key = (self.issuer, attestation_id, nonce) + if key in self._consumed: + raise CandidateArtifactProvenanceError("candidate attestation is replayed") + self._consumed.add(key) + + +@dataclass(frozen=True) +class CandidateArtifactProvenance: + """A signed build statement bound to one immutable candidate wheel.""" + + issuer: str + attestation_id: str + nonce: str + issued_at: str + expires_at: str + wheel_sha256: str + source_sha: str + dependency_lock_sha256: str + python: dict[str, str] + builder_workflow_identity: str + signature: str + + @classmethod + def from_payload(cls, payload: object) -> "CandidateArtifactProvenance": + """Parse one untrusted JSON build attestation strictly.""" + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise CandidateArtifactProvenanceError("candidate attestation schema is invalid") + signature = payload.get("signature") + fields = ( + "issuer", "attestation_id", "nonce", "issued_at", "expires_at", + "wheel_sha256", "source_sha", "dependency_lock_sha256", + "builder_workflow_identity", + ) + if ( + not isinstance(signature, dict) + or signature.get("algorithm") != "Ed25519" + or not isinstance(signature.get("value"), str) + or any(not isinstance(payload.get(name), str) or not payload[name] for name in fields) + or not isinstance(payload.get("python"), dict) + ): + raise CandidateArtifactProvenanceError("candidate attestation is malformed") + python = payload["python"] + expected_python = {"implementation", "version", "abi", "platform"} + if set(python) != expected_python or any( + not isinstance(python[name], str) or not python[name] for name in expected_python + ): + raise CandidateArtifactProvenanceError("candidate attestation interpreter is invalid") + _sha256_value(payload["wheel_sha256"], "wheel digest") + _sha256_value(payload["dependency_lock_sha256"], "dependency lock digest") + source_sha = payload["source_sha"] + if len(source_sha) != 40 or any( + character not in "0123456789abcdef" for character in source_sha + ): + raise CandidateArtifactProvenanceError("candidate attestation source SHA is invalid") + _timestamp(payload["issued_at"], "issued_at") + _timestamp(payload["expires_at"], "expires_at") + return cls( + *(payload[name] for name in fields[:8]), + dict(python), + payload["builder_workflow_identity"], + signature["value"], + ) + + def unsigned_payload(self) -> dict[str, Any]: + """Return the exact canonical payload protected by the builder signature.""" + return { + "schema_version": 1, + "issuer": self.issuer, + "attestation_id": self.attestation_id, + "nonce": self.nonce, + "issued_at": self.issued_at, + "expires_at": self.expires_at, + "wheel_sha256": self.wheel_sha256, + "source_sha": self.source_sha, + "dependency_lock_sha256": self.dependency_lock_sha256, + "python": self.python, + "builder_workflow_identity": self.builder_workflow_identity, + } + + def payload(self) -> dict[str, Any]: + """Return the serializable signed statement for certificate inclusion.""" + return { + **self.unsigned_payload(), + "signature": {"algorithm": "Ed25519", "value": self.signature}, + } + + def verify( + self, + policy: CandidateArtifactPolicy, + *, + expected_source_sha: str | None, + now: datetime | None = None, + consume_replay: bool = True, + ) -> None: + """Verify signature, freshness, protected build environment, and source.""" + if self.issuer != policy.issuer: + raise CandidateArtifactProvenanceError( + "candidate attestation issuer is untrusted" + ) + try: + signature = base64.b64decode(self.signature, validate=True) + Ed25519PublicKey.from_public_bytes(policy.public_key).verify( + signature, _canonical_bytes(self.unsigned_payload()) + ) + except (ValueError, InvalidSignature) as exc: + raise CandidateArtifactProvenanceError( + "candidate attestation signature is invalid" + ) from exc + issued = _timestamp(self.issued_at, "issued_at") + expires = _timestamp(self.expires_at, "expires_at") + current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + if expires <= issued or expires <= current: + raise CandidateArtifactProvenanceError("candidate attestation is expired") + if issued > current + timedelta(minutes=5): + raise CandidateArtifactProvenanceError("candidate attestation is not yet valid") + if expected_source_sha is not None and self.source_sha != expected_source_sha: + raise CandidateArtifactProvenanceError( + "candidate attestation source SHA does not match certified PDD head" + ) + if self.builder_workflow_identity != policy.builder_workflow_identity: + raise CandidateArtifactProvenanceError( + "candidate attestation builder workflow is not protected" + ) + if self.dependency_lock_sha256 != policy.dependency_lock_sha256: + raise CandidateArtifactProvenanceError( + "candidate attestation dependency lock does not match" + ) + expected_python = { + "implementation": policy.python_implementation, + "version": policy.python_version, + "abi": policy.python_abi, + "platform": policy.python_platform, + } + if self.python != expected_python: + raise CandidateArtifactProvenanceError( + "candidate attestation interpreter does not match" + ) + if consume_replay: + policy.consume(self.attestation_id, self.nonce) + + +def load_candidate_artifact_provenance( + path: Path, + wheel: Path, + policy: CandidateArtifactPolicy, +) -> CandidateArtifactProvenance: + """Load an untrusted statement and bind it to exact wheel bytes before use.""" + attestation = Path(path) + candidate = Path(wheel) + if candidate.is_symlink() or not candidate.is_file() or candidate.suffix != ".whl": + raise CandidateArtifactProvenanceError("candidate wheel path is unsafe") + if attestation.is_symlink() or not attestation.is_file(): + raise CandidateArtifactProvenanceError("candidate attestation path is unsafe") + try: + provenance = CandidateArtifactProvenance.from_payload( + json.loads(attestation.read_text(encoding="utf-8")) + ) + except (OSError, json.JSONDecodeError) as exc: + raise CandidateArtifactProvenanceError("candidate attestation cannot be read") from exc + if _sha256(candidate) != provenance.wheel_sha256: + raise CandidateArtifactProvenanceError("candidate wheel digest does not match attestation") + provenance.verify(policy, expected_source_sha=None, consume_replay=False) + return provenance + + +def candidate_artifact_policy_from_environment() -> CandidateArtifactPolicy: + """Load only protected-CI inputs required to authorize a candidate build.""" + names = { + "issuer": "PDD_CANDIDATE_ATTESTATION_ISSUER", + "public_key": "PDD_CANDIDATE_ATTESTATION_PUBLIC_KEY", + "builder_workflow_identity": "PDD_CANDIDATE_BUILDER_WORKFLOW_IDENTITY", + "dependency_lock_sha256": "PDD_CANDIDATE_DEPENDENCY_LOCK_SHA256", + "python_implementation": "PDD_CANDIDATE_PYTHON_IMPLEMENTATION", + "python_version": "PDD_CANDIDATE_PYTHON_VERSION", + "python_abi": "PDD_CANDIDATE_PYTHON_ABI", + "python_platform": "PDD_CANDIDATE_PYTHON_PLATFORM", + } + values = {name: os.environ.get(env_name, "") for name, env_name in names.items()} + if not all(values.values()): + raise CandidateArtifactProvenanceError( + "protected candidate artifact provenance is required" + ) + try: + values["public_key"] = base64.b64decode(values["public_key"], validate=True) + except ValueError as exc: + raise CandidateArtifactProvenanceError( + "candidate artifact public key is malformed" + ) from exc + return CandidateArtifactPolicy(**values) diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index f45f7fd3a..cf22c80fd 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -1,4 +1,5 @@ """Cross-repository signed Global Sync Certificate aggregation.""" +# pylint: disable=too-many-lines from __future__ import annotations @@ -9,7 +10,7 @@ import subprocess import fnmatch import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone from pathlib import Path, PurePosixPath from typing import Any, Protocol @@ -17,6 +18,11 @@ from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from .artifact_provenance import ( + CandidateArtifactPolicy, + CandidateArtifactProvenance, + CandidateArtifactProvenanceError, +) from .reporting import CanonicalReportOptions, build_canonical_report @@ -88,6 +94,7 @@ class LifecycleResult: missing_scenarios: tuple[str, ...] = () candidate_wheel_sha256: str = "" dependency_environment_digest: str = "" + candidate_artifact: CandidateArtifactProvenance | None = None @dataclass(frozen=True) @@ -149,6 +156,7 @@ class GlobalCertificateOptions: required_nightly_streak: int = 7 checker_identity: CheckerIdentity | None = None nightly_observation: NightlyObservation | None = None + candidate_artifact_policy: CandidateArtifactPolicy | None = None def _canonical_bytes(payload: dict[str, Any]) -> bytes: @@ -398,6 +406,7 @@ def _complete_nightly( "timeouts", "candidate_wheel_sha256", "dependency_environment_digest", + "candidate_artifact", } return ( required_counts <= counts.keys() @@ -534,6 +543,7 @@ def _scan_predicate( character in "0123456789abcdef" for character in lifecycle.dependency_environment_digest ) + and lifecycle.candidate_artifact is not None and extra["nightly_observation_complete"] == 1 ) @@ -605,8 +615,8 @@ def _canonical_report_predicate(report: dict[str, Any]) -> bool: def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool]: - # pylint: disable=too-many-locals,too-many-return-statements - if body.get("schema_version") != 2: + # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches + if body.get("schema_version") != 3: return False, False checker = body.get("checker") try: @@ -669,10 +679,33 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] for value in digests.values() ): return False, False + candidate_artifact_payload = lifecycle_payload.get("candidate_artifact") + try: + candidate_artifact = CandidateArtifactProvenance.from_payload( + candidate_artifact_payload + ) + except (CandidateArtifactProvenanceError, ValueError): + return False, False + if candidate_artifact.wheel_sha256 != digests["candidate_wheel_sha256"]: + return False, False + pdd_report = next( + ( + item + for item in reports + if isinstance(item, dict) and item.get("repository") == "pdd" + ), + None, + ) + if ( + not isinstance(pdd_report, dict) + or candidate_artifact.source_sha != pdd_report.get("head_sha") + ): + return False, False lifecycle = LifecycleResult( *(lifecycle_payload[name] for name in lifecycle_names), tuple(lifecycle_payload["missing_scenarios"]), *(digests[name] for name in digest_names), + candidate_artifact, ) extra_names = { "pdd_cloud_vendored_sync_semantics", @@ -727,6 +760,8 @@ def _build_global_certificate_from_targets( raise ValueError("global certificate requires at least one repository") if options.checker_identity is None: raise ValueError("global certificate requires released checker identity") + if options.candidate_artifact_policy is None: + raise ValueError("global certificate requires candidate artifact policy") reports = [] for target in targets: report = build_canonical_report( @@ -739,6 +774,17 @@ def _build_global_certificate_from_targets( ) reports.append({**report, "repository": target.name}) counts = _aggregate_counts(reports) + pdd_report = next(report for report in reports if report["repository"] == "pdd") + candidate_artifact_valid = False + if options.lifecycle_result.candidate_artifact is not None: + try: + options.lifecycle_result.candidate_artifact.verify( + options.candidate_artifact_policy, + expected_source_sha=str(pdd_report.get("head_sha", "")), + ) + candidate_artifact_valid = True + except CandidateArtifactProvenanceError: + candidate_artifact_valid = False cloud = next((target for target in targets if target.name == "pdd_cloud"), None) extra = { "pdd_cloud_vendored_sync_semantics": ( @@ -761,11 +807,16 @@ def _build_global_certificate_from_targets( and _nightly_observation_complete(options.nightly_observation.payload()) ), } - lifecycle = options.lifecycle_result + lifecycle = ( + options.lifecycle_result + if candidate_artifact_valid + else replace(options.lifecycle_result, candidate_artifact=None) + ) body: dict[str, Any] = { - "schema_version": 2, + "schema_version": 3, "checked_at": datetime.now(timezone.utc).isoformat(), "checker": options.checker_identity.payload(), + "candidate_artifact_policy": options.candidate_artifact_policy.identity(), "nightly_observation": ( options.nightly_observation.payload() if options.nightly_observation is not None @@ -798,6 +849,11 @@ def _build_global_certificate_from_targets( "missing_scenarios": list(lifecycle.missing_scenarios), "candidate_wheel_sha256": lifecycle.candidate_wheel_sha256, "dependency_environment_digest": lifecycle.dependency_environment_digest, + "candidate_artifact": ( + lifecycle.candidate_artifact.payload() + if lifecycle.candidate_artifact is not None and candidate_artifact_valid + else None + ), }, } body["scan_ok"] = all(report.get("ok") for report in reports) and _scan_predicate( @@ -871,10 +927,11 @@ def verify_global_certificate( expected_repository_shas: dict[str, tuple[str, str]], expected_repository_ids: dict[str, str], expected_checker_identity: CheckerIdentity, + expected_candidate_artifact_policy: CandidateArtifactPolicy, now: datetime | None = None, maximum_age: timedelta = timedelta(minutes=15), ) -> bool: - # pylint: disable=too-many-arguments,too-many-return-statements + # pylint: disable=too-many-arguments,too-many-return-statements,too-many-locals """Accept only a fresh green certificate for exact expected repository refs.""" if not _verify_certificate_integrity( certificate, public_key, expected_issuer=expected_issuer @@ -884,6 +941,34 @@ def verify_global_certificate( return False if certificate.get("checker") != expected_checker_identity.payload(): return False + if ( + certificate.get("candidate_artifact_policy") + != expected_candidate_artifact_policy.identity() + ): + return False + lifecycle = certificate.get("lifecycle") + reports = certificate.get("repositories") + if not isinstance(lifecycle, dict) or not isinstance(reports, list): + return False + pdd_report = next( + ( + item + for item in reports + if isinstance(item, dict) and item.get("repository") == "pdd" + ), + None, + ) + try: + artifact = CandidateArtifactProvenance.from_payload( + lifecycle.get("candidate_artifact") + ) + artifact.verify( + expected_candidate_artifact_policy, + expected_source_sha=str(pdd_report.get("head_sha", "")), + consume_replay=False, + ) + except (CandidateArtifactProvenanceError, AttributeError, ValueError): + return False try: checked_at = datetime.fromisoformat(str(certificate["checked_at"])) except (KeyError, ValueError): @@ -894,7 +979,6 @@ def verify_global_certificate( checked_at = checked_at.astimezone(timezone.utc) if checked_at > current or current - checked_at > maximum_age: return False - reports = certificate.get("repositories") if not isinstance(reports, list) or len(reports) != len(expected_repository_shas): return False actual = { diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 6c800ac48..10d0ed8ac 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -11,6 +11,11 @@ from pathlib import Path from typing import Any +from .artifact_provenance import ( + CandidateArtifactPolicy, + CandidateArtifactProvenanceError, + load_candidate_artifact_provenance, +) from .certificate import LifecycleResult from .isolation import untrusted_child_environment from .scenario_contract import REQUIRED_SCENARIOS @@ -36,6 +41,7 @@ def _failed_result(*, timeout: bool = False) -> LifecycleResult: tuple(sorted(REQUIRED_SCENARIOS)), "", "", + None, ) @@ -206,12 +212,14 @@ def run_lifecycle_matrix( candidate_wheel: Path | None = None, candidate_wheelhouse: Path | None = None, candidate_runtime_lock: Path | None = None, + candidate_attestation: Path | None = None, + candidate_artifact_policy: CandidateArtifactPolicy | None = None, cloud_root: Path | None = None, cloud_base_ref: str | None = None, cloud_head_ref: str | None = None, timeout_seconds: int = 1200, ) -> LifecycleResult: - # pylint: disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals,too-many-boolean-expressions """Run only the scenario harness installed with the released checker.""" del root # Candidate repository tests are never lifecycle evidence. required_paths = ( @@ -221,11 +229,19 @@ def run_lifecycle_matrix( ) if ( not _required_paths_available(required_paths) + or candidate_attestation is None + or candidate_artifact_policy is None or cloud_root is None or cloud_base_ref is None or cloud_head_ref is None ): return _failed_result() + try: + candidate_artifact = load_candidate_artifact_provenance( + candidate_attestation, Path(candidate_wheel), candidate_artifact_policy + ) + except CandidateArtifactProvenanceError: + return _failed_result() with tempfile.TemporaryDirectory(prefix="pdd-released-lifecycle-") as directory: temporary = Path(directory) output = temporary / "result.json" @@ -297,4 +313,5 @@ def run_lifecycle_matrix( missing, hashlib.sha256(Path(candidate_wheel).read_bytes()).hexdigest(), dependency_digest, + candidate_artifact, ) diff --git a/tests/test_sync_core_candidate_artifact_provenance.py b/tests/test_sync_core_candidate_artifact_provenance.py index 845d07d97..89e5bbfb5 100644 --- a/tests/test_sync_core_candidate_artifact_provenance.py +++ b/tests/test_sync_core_candidate_artifact_provenance.py @@ -1,4 +1,5 @@ """Strict tests for protected candidate-wheel build provenance.""" +# pylint: disable=missing-function-docstring,line-too-long import hashlib import json @@ -88,7 +89,9 @@ def test_correct_wheel_with_wrong_certified_source_sha_is_rejected(tmp_path) -> wheel = tmp_path / "candidate.whl" wheel.write_bytes(b"exact wheel") with pytest.raises(CandidateArtifactProvenanceError, match="source SHA"): - _load(tmp_path, wheel, authority, source_sha="c" * 40) + _load(tmp_path, wheel, authority, source_sha="c" * 40).verify( + _policy(authority), expected_source_sha=SOURCE_SHA + ) def test_forged_build_attestation_is_rejected(tmp_path) -> None: @@ -115,9 +118,11 @@ def test_stale_or_replayed_build_attestation_is_rejected(tmp_path) -> None: issued_at=(datetime.now(timezone.utc) - timedelta(hours=2)).isoformat(), expires_at=(datetime.now(timezone.utc) - timedelta(hours=1)).isoformat(), ) + policy = _policy(authority) provenance = _load(tmp_path, wheel, authority) + provenance.verify(policy, expected_source_sha=SOURCE_SHA) with pytest.raises(CandidateArtifactProvenanceError, match="replayed"): - provenance.verify(_policy(authority), expected_source_sha=SOURCE_SHA) + provenance.verify(policy, expected_source_sha=SOURCE_SHA) @pytest.mark.parametrize( @@ -133,15 +138,21 @@ def test_wrong_protected_build_environment_is_rejected(tmp_path, field, value, e wheel = tmp_path / "candidate.whl" wheel.write_bytes(b"exact wheel") with pytest.raises(CandidateArtifactProvenanceError, match=error): - _load(tmp_path, wheel, authority, **{field: value}) + _load(tmp_path, wheel, authority, **{field: value}).verify( + _policy(authority), expected_source_sha=SOURCE_SHA + ) def test_source_checkout_is_not_a_candidate_artifact(tmp_path) -> None: authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") checkout = tmp_path / "pdd" checkout.mkdir() + path = tmp_path / "candidate-build-attestation.json" + path.write_text(json.dumps(_attestation(wheel, authority))) with pytest.raises(CandidateArtifactProvenanceError, match="wheel path"): - _load(tmp_path, checkout, authority) + load_candidate_artifact_provenance(path, checkout, _policy(authority)) def test_valid_exact_sha_artifact_is_accepted(tmp_path) -> None: @@ -149,5 +160,6 @@ def test_valid_exact_sha_artifact_is_accepted(tmp_path) -> None: wheel = tmp_path / "candidate.whl" wheel.write_bytes(b"exact wheel") provenance = _load(tmp_path, wheel, authority) + provenance.verify(_policy(authority), expected_source_sha=SOURCE_SHA) assert provenance.source_sha == SOURCE_SHA assert provenance.wheel_sha256 == hashlib.sha256(wheel.read_bytes()).hexdigest() diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 4b977835a..4c0da86e7 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -9,6 +9,8 @@ from pdd.sync_core import ( AttestationSigner, + CandidateArtifactPolicy, + CandidateArtifactProvenance, CheckerIdentity, GlobalCertificateOptions, LifecycleResult, @@ -31,6 +33,17 @@ OBSERVATION = NightlyObservation(True, True, 0, True, "BLOCKED", 0) CANDIDATE_WHEEL_SHA256 = "d" * 64 DEPENDENCY_ENVIRONMENT_DIGEST = "e" * 64 +CANDIDATE_BUILDER = AttestationSigner("candidate-builder", b"h" * 32) +CANDIDATE_POLICY = CandidateArtifactPolicy( + CANDIDATE_BUILDER.issuer, + CANDIDATE_BUILDER.public_key_bytes(), + "promptdriven/pdd/.github/workflows/candidate-wheel.yml@refs/heads/main", + DEPENDENCY_ENVIRONMENT_DIGEST, + "CPython", + "3.12.3", + "cp312", + "manylinux_2_17_x86_64", +) @pytest.fixture(autouse=True) @@ -112,11 +125,13 @@ def _nightly(path, signer, count=7, *, include_observation=True): "missing_scenarios": [], "candidate_wheel_sha256": CANDIDATE_WHEEL_SHA256, "dependency_environment_digest": DEPENDENCY_ENVIRONMENT_DIGEST, + "candidate_artifact": _candidate_artifact().payload(), } body = { - "schema_version": 2, + "schema_version": 3, "checked_at": (start + timedelta(days=index)).isoformat(), "checker": CHECKER.payload(), + "candidate_artifact_policy": CANDIDATE_POLICY.identity(), "repositories": reports, "counts": counts, "lifecycle": lifecycle, @@ -149,6 +164,44 @@ def _expected_ids(certificate): } +def _candidate_artifact(source_sha="b" * 40): + now = datetime.now(timezone.utc) + artifact = CandidateArtifactProvenance( + CANDIDATE_BUILDER.issuer, + f"candidate-build-{now.timestamp()}", + f"candidate-build-nonce-{now.timestamp()}", + now.isoformat(), + (now + timedelta(minutes=10)).isoformat(), + CANDIDATE_WHEEL_SHA256, + source_sha, + DEPENDENCY_ENVIRONMENT_DIGEST, + { + "implementation": CANDIDATE_POLICY.python_implementation, + "version": CANDIDATE_POLICY.python_version, + "abi": CANDIDATE_POLICY.python_abi, + "platform": CANDIDATE_POLICY.python_platform, + }, + CANDIDATE_POLICY.builder_workflow_identity, + "", + ) + unsigned = {key: value for key, value in artifact.payload().items() if key != "signature"} + return CandidateArtifactProvenance( + artifact.issuer, + artifact.attestation_id, + artifact.nonce, + artifact.issued_at, + artifact.expires_at, + artifact.wheel_sha256, + artifact.source_sha, + artifact.dependency_lock_sha256, + artifact.python, + artifact.builder_workflow_identity, + CANDIDATE_BUILDER.sign_bytes( + json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode() + ), + ) + + def test_lifecycle_child_environment_excludes_all_signing_material( tmp_path, monkeypatch ) -> None: @@ -221,6 +274,7 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, + candidate_artifact=_candidate_artifact(), ) certificate = build_global_certificate( ( @@ -233,6 +287,7 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, + candidate_artifact_policy=CANDIDATE_POLICY, ), signer=signer, ) @@ -251,6 +306,7 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc expected_repository_shas=expected, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is True assert ( certificate["lifecycle"]["candidate_wheel_sha256"] @@ -268,6 +324,29 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc expected_repository_shas=expected, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, + ) is False + certificate["lifecycle"]["candidate_wheel_sha256"] = CANDIDATE_WHEEL_SHA256 + forged_artifact = certificate["lifecycle"]["candidate_artifact"] + forged_artifact["source_sha"] = "c" * 40 + unsigned_artifact = { + key: value for key, value in forged_artifact.items() if key != "signature" + } + forged_artifact["signature"]["value"] = CANDIDATE_BUILDER.sign_bytes( + json.dumps(unsigned_artifact, sort_keys=True, separators=(",", ":")).encode() + ) + body = {key: value for key, value in certificate.items() if key != "signature"} + certificate["signature"]["value"] = signer.sign_bytes( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ) + assert verify_global_certificate( + certificate, + signer.public_key_bytes(), + expected_issuer=signer.issuer, + expected_repository_shas=expected, + expected_repository_ids=_expected_ids(certificate), + expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False certificate["counts"]["managed_units"] = 0 assert verify_global_certificate( @@ -277,6 +356,7 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc expected_repository_shas=expected, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False @@ -306,6 +386,7 @@ def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> ("merge-group",), CANDIDATE_WHEEL_SHA256, DEPENDENCY_ENVIRONMENT_DIGEST, + _candidate_artifact(), ) certificate = build_global_certificate( ( @@ -318,6 +399,7 @@ def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, + candidate_artifact_policy=CANDIDATE_POLICY, ), signer=signer, ) @@ -333,6 +415,7 @@ def test_missing_lifecycle_scenario_blocks_certificate(tmp_path, monkeypatch) -> expected_repository_shas=expected, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False @@ -371,10 +454,12 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, + candidate_artifact=_candidate_artifact(), ), nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, + candidate_artifact_policy=CANDIDATE_POLICY, ), signer=signer, ) @@ -388,6 +473,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( "expected_repository_shas": expected, "expected_repository_ids": _expected_ids(certificate), "expected_checker_identity": CHECKER, + "expected_candidate_artifact_policy": CANDIDATE_POLICY, } assert verify_global_certificate( certificate, signer.public_key_bytes(), **common @@ -404,6 +490,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( expected_repository_shas=expected, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=other_checker, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False wrong_ids = {**_expected_ids(certificate), "pdd_cloud": "wrong-repository"} assert verify_global_certificate( @@ -413,6 +500,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( expected_repository_shas=expected, expected_repository_ids=wrong_ids, expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False assert verify_global_certificate( certificate, @@ -427,6 +515,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( expected_repository_shas=expected, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False wrong_refs = {**expected, "pdd_cloud": ("a" * 40, "c" * 40)} assert verify_global_certificate( @@ -436,6 +525,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( expected_repository_shas=wrong_refs, expected_repository_ids=_expected_ids(certificate), expected_checker_identity=CHECKER, + expected_candidate_artifact_policy=CANDIDATE_POLICY, ) is False @@ -525,10 +615,12 @@ def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, + candidate_artifact=_candidate_artifact(), ), nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, + candidate_artifact_policy=CANDIDATE_POLICY, ), signer=AttestationSigner("certificate-ci", b"g" * 32), ) @@ -571,10 +663,12 @@ def test_signed_nightly_rows_without_observations_do_not_extend_streak( 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, + candidate_artifact=_candidate_artifact(), ), nightly, checker_identity=CHECKER, nightly_observation=OBSERVATION, + candidate_artifact_policy=CANDIDATE_POLICY, ), signer=signer, ) From 4a30a977dadc1ec214002a4a09ede0e52a35cabb Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 22:54:37 -0700 Subject: [PATCH 027/128] fix(sync): verify candidate provenance in certificate paths --- pdd/sync_core/artifact_provenance.py | 32 ++++++++++++ pdd/sync_core/certificate.py | 74 +++++++++++++++++++-------- pdd/sync_core/certificate_verifier.py | 31 ++++++++++- 3 files changed, 116 insertions(+), 21 deletions(-) diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py index d29dab56c..ddd5c6e63 100644 --- a/pdd/sync_core/artifact_provenance.py +++ b/pdd/sync_core/artifact_provenance.py @@ -70,6 +70,7 @@ class CandidateArtifactPolicy: python_version: str python_abi: str python_platform: str + replay_ledger_path: Path | None = None _consumed: set[tuple[str, str, str]] = field(default_factory=set, init=False) def __post_init__(self) -> None: @@ -100,10 +101,41 @@ def identity(self) -> dict[str, str]: def consume(self, attestation_id: str, nonce: str) -> None: """Reject a build statement reused by a second certification attempt.""" key = (self.issuer, attestation_id, nonce) + if self.replay_ledger_path is not None: + self._consume_durable(key) + return if key in self._consumed: raise CandidateArtifactProvenanceError("candidate attestation is replayed") self._consumed.add(key) + def _consume_durable(self, key: tuple[str, str, str]) -> None: + path = Path(self.replay_ledger_path).expanduser().resolve() + if path.is_symlink(): + raise CandidateArtifactProvenanceError("candidate replay ledger is unsafe") + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + try: + records = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise CandidateArtifactProvenanceError( + "candidate replay ledger is corrupt" + ) from exc + if not isinstance(records, list) or any( + not isinstance(item, list) + or len(item) != 3 + or any(not isinstance(value, str) for value in item) + for item in records + ): + raise CandidateArtifactProvenanceError( + "candidate replay ledger is corrupt" + ) + else: + records = [] + if list(key) in records: + raise CandidateArtifactProvenanceError("candidate attestation is replayed") + records.append(list(key)) + path.write_text(json.dumps(records, sort_keys=True) + "\n", encoding="utf-8") + @dataclass(frozen=True) class CandidateArtifactProvenance: diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index cf22c80fd..aa9b49817 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -146,6 +146,17 @@ def payload(self) -> dict[str, bool | int | str]: } +@dataclass(frozen=True) +class _NightlyVerificationPolicy: + """Protected inputs required to accept one historical nightly row.""" + + public_key: bytes + expected_issuer: str + targets: tuple[RepositoryTarget, ...] + checker_identity: CheckerIdentity + candidate_artifact_policy: CandidateArtifactPolicy + + @dataclass(frozen=True) class GlobalCertificateOptions: """Trust and external evidence inputs for global certification.""" @@ -377,10 +388,7 @@ def _nightly_lineage( def _complete_nightly( row: Any, - public_key: bytes, - expected_issuer: str, - targets: tuple[RepositoryTarget, ...], - checker_identity: CheckerIdentity, + policy: _NightlyVerificationPolicy, ) -> bool: if not isinstance(row, dict) or row.get("scan_ok") is not True: return False @@ -408,16 +416,44 @@ def _complete_nightly( "dependency_environment_digest", "candidate_artifact", } - return ( + if not ( required_counts <= counts.keys() and required_lifecycle <= lifecycle.keys() and _nightly_observation_complete(row.get("nightly_observation")) - and row.get("checker") == checker_identity.payload() - and _nightly_lineage(row, targets) + and row.get("checker") == policy.checker_identity.payload() + and _nightly_lineage(row, policy.targets) and _verify_certificate_integrity( - row, public_key, expected_issuer=expected_issuer + row, policy.public_key, expected_issuer=policy.expected_issuer ) - ) + ): + return False + return _nightly_candidate_artifact_valid(repositories, lifecycle, policy) + + +def _nightly_candidate_artifact_valid( + repositories: list[Any], + lifecycle: dict[str, Any], + policy: _NightlyVerificationPolicy, +) -> bool: + """Verify the candidate artifact embedded in one historical nightly row.""" + by_name = { + item.get("repository"): item for item in repositories if isinstance(item, dict) + } + pdd_report = by_name.get("pdd") + if not isinstance(pdd_report, dict): + return False + try: + artifact = CandidateArtifactProvenance.from_payload( + lifecycle.get("candidate_artifact") + ) + artifact.verify( + policy.candidate_artifact_policy, + expected_source_sha=str(pdd_report.get("head_sha", "")), + consume_replay=False, + ) + except (CandidateArtifactProvenanceError, ValueError): + return False + return artifact.wheel_sha256 == lifecycle.get("candidate_wheel_sha256") def _nightly_observation_complete(payload: Any) -> bool: @@ -438,10 +474,7 @@ def _nightly_observation_complete(payload: Any) -> bool: def _nightly_streak( path: Path, - public_key: bytes, - expected_issuer: str, - targets: tuple[RepositoryTarget, ...], - checker_identity: CheckerIdentity, + policy: _NightlyVerificationPolicy, ) -> int: if not path.exists(): return 0 @@ -455,9 +488,7 @@ def _nightly_streak( previous_date = None today = datetime.now(timezone.utc).date() for row in reversed(rows): - if not _complete_nightly( - row, public_key, expected_issuer, targets, checker_identity - ): + if not _complete_nightly(row, policy): break try: checked_at = datetime.fromisoformat(str(row["checked_at"])) @@ -796,10 +827,13 @@ def _build_global_certificate_from_targets( ), "nightly_streak": _nightly_streak( options.nightly_ledger, - signer.public_key_bytes(), - signer.issuer, - targets, - options.checker_identity, + _NightlyVerificationPolicy( + signer.public_key_bytes(), + signer.issuer, + targets, + options.checker_identity, + options.candidate_artifact_policy, + ), ), "required_nightly_streak": options.required_nightly_streak, "nightly_observation_complete": int( diff --git a/pdd/sync_core/certificate_verifier.py b/pdd/sync_core/certificate_verifier.py index cd8e8f371..61e937b03 100644 --- a/pdd/sync_core/certificate_verifier.py +++ b/pdd/sync_core/certificate_verifier.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from .artifact_provenance import CandidateArtifactPolicy from .certificate import CheckerIdentity, verify_global_certificate @@ -19,6 +20,7 @@ class CertificateExpectations: issuer: str public_key: bytes checker: CheckerIdentity + candidate_artifact_policy: CandidateArtifactPolicy repository_shas: dict[str, tuple[str, str]] repository_ids: dict[str, str] @@ -30,6 +32,25 @@ def _sha(value: Any) -> str: return text +def _candidate_policy(payload: Any) -> CandidateArtifactPolicy: + if not isinstance(payload, dict): + raise ValueError("candidate artifact policy is malformed") + python = payload.get("python") + if not isinstance(python, dict): + raise ValueError("candidate artifact policy interpreter is malformed") + public_key = base64.b64decode(str(payload["public_key_base64"]), validate=True) + return CandidateArtifactPolicy( + str(payload["issuer"]), + public_key, + str(payload["builder_workflow_identity"]), + str(payload["dependency_lock_sha256"]), + str(python["implementation"]), + str(python["version"]), + str(python["abi"]), + str(python["platform"]), + ) + + def load_expectations(path: Path) -> CertificateExpectations: """Load and strictly validate one protected expectations document.""" try: @@ -37,6 +58,7 @@ def load_expectations(path: Path) -> CertificateExpectations: if not isinstance(payload, dict) or payload.get("schema_version") != 1: raise ValueError("expectations schema is invalid") checker_payload = payload["checker"] + candidate_policy_payload = payload["candidate_artifact_policy"] repositories = payload["repositories"] if not isinstance(checker_payload, dict) or not isinstance(repositories, dict): raise ValueError("expectations payload is malformed") @@ -53,6 +75,7 @@ def load_expectations(path: Path) -> CertificateExpectations: issuer = str(payload["issuer"]) if not issuer: raise ValueError("certificate issuer is absent") + candidate_policy = _candidate_policy(candidate_policy_payload) repository_shas: dict[str, tuple[str, str]] = {} repository_ids: dict[str, str] = {} for name, row in repositories.items(): @@ -63,7 +86,12 @@ def load_expectations(path: Path) -> CertificateExpectations: except (KeyError, TypeError, json.JSONDecodeError, OSError) as exc: raise ValueError("protected certificate expectations are malformed") from exc return CertificateExpectations( - issuer, public_key, checker, repository_shas, repository_ids + issuer, + public_key, + checker, + candidate_policy, + repository_shas, + repository_ids, ) @@ -85,6 +113,7 @@ def main() -> None: expected_repository_shas=expected.repository_shas, expected_repository_ids=expected.repository_ids, expected_checker_identity=expected.checker, + expected_candidate_artifact_policy=expected.candidate_artifact_policy, ) except (OSError, ValueError, json.JSONDecodeError): verified = False From 12e4743e80d6b1fcb56b3d7ad41171fed420e9f7 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 06:40:09 -0700 Subject: [PATCH 028/128] fix(sync): isolate protected pytest execution --- pdd/sync_core/runner.py | 133 ++++++++++++++++++++++++++++----- pdd/sync_core/types.py | 3 + pdd/sync_core/verification.py | 7 ++ tests/test_sync_core_runner.py | 53 ++++--------- 4 files changed, 137 insertions(+), 59 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index c41235482..e910f5d2c 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -110,23 +110,13 @@ def _declares_pytest_plugins(targets: tuple[ast.AST, ...]) -> bool: ) -def _is_pytest_support_path(path: PurePosixPath) -> bool: - """Return whether a path is pytest support rather than product code.""" - return ( - path.name == "conftest.py" - or path in PYTEST_CONFIG_PATHS - or "tests" in path.parts - or path.name.startswith("test_") - ) - - def _local_module_paths( root: Path, ref: str, source_path: PurePosixPath, source: bytes, *, - support_only: bool = False, + code_under_test_paths: frozenset[PurePosixPath] = frozenset(), ) -> tuple[set[PurePosixPath], bool]: # pylint: disable=too-many-locals """Resolve repository-local Python imports without executing candidate code.""" @@ -171,13 +161,16 @@ def _local_module_paths( path for path in candidates if read_git_blob(root, ref, path) is not None - and (not support_only or _is_pytest_support_path(path)) + and path not in code_under_test_paths ) return resolved, dynamic_pytest_plugins def _pytest_support_closure( - root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] + root: Path, + ref: str, + test_paths: tuple[PurePosixPath, ...], + code_under_test_paths: tuple[PurePosixPath, ...] = (), ) -> tuple[tuple[PurePosixPath, bytes], ...]: """Return config, conftest, and transitive local-import blobs for pytest.""" config_paths = _pytest_config_paths(root, ref, test_paths) @@ -187,6 +180,7 @@ def _pytest_support_closure( pending=tuple(test_paths) + tuple(config_paths), included=config_paths, test_artifacts=test_paths, + code_under_test_paths=frozenset(code_under_test_paths), ) @@ -218,6 +212,7 @@ def _transitive_support_blobs( pending: tuple[PurePosixPath, ...], included: set[PurePosixPath], test_artifacts: tuple[PurePosixPath, ...] = (), + code_under_test_paths: frozenset[PurePosixPath] = frozenset(), ) -> tuple[tuple[PurePosixPath, bytes], ...]: """Resolve local Python imports from protected runner support paths.""" paths = set(included) @@ -237,7 +232,9 @@ def _transitive_support_blobs( ref, path, source, - support_only=path in test_artifact_set, + code_under_test_paths=( + code_under_test_paths if path in test_artifact_set else frozenset() + ), ) discovered -= visited paths.update(discovered) @@ -269,7 +266,6 @@ def _has_dynamic_pytest_plugins( """Fail closed when pytest plugin declarations cannot be statically bound.""" config_paths = _pytest_config_paths(root, ref, test_paths) remaining = list(tuple(test_paths) + tuple(config_paths)) - test_artifact_set = set(test_paths) visited: set[PurePosixPath] = set() while remaining: path = remaining.pop() @@ -284,7 +280,6 @@ def _has_dynamic_pytest_plugins( ref, path, source, - support_only=path in test_artifact_set, ) if dynamic: return True @@ -292,6 +287,35 @@ def _has_dynamic_pytest_plugins( return False +def _has_external_pytest_plugins( + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] +) -> bool: + """Return whether a literal plugin declaration lacks protected repo bytes.""" + for path in tuple(test_paths) + tuple(_pytest_config_paths(root, ref, test_paths)): + source = read_git_blob(root, ref, path) + if source is None or path.suffix != ".py": + continue + try: + tree = ast.parse(source) + except (SyntaxError, UnicodeDecodeError): + continue + for node in ast.walk(tree): + if not _declares_pytest_plugins(_pytest_plugin_declaration_targets(node)): + continue + modules, dynamic = _pytest_plugin_modules(node.value) + if dynamic: + continue + for module in modules: + module_path = PurePosixPath(*module.split(".")) + candidates = ( + module_path.with_suffix(".py"), + module_path / "__init__.py", + ) + if not any(read_git_blob(root, ref, item) is not None for item in candidates): + return True + return False + + def _support_digest( root: Path, ref: str, profile: VerificationProfile ) -> tuple[str, tuple[PurePosixPath, ...]]: @@ -301,7 +325,11 @@ def _support_digest( if obligation.validator_id == "pytest" for path in obligation.artifact_paths ) - closure = _pytest_support_closure(root, ref, tests) + product = tuple( + path for obligation in profile.obligations + for path in obligation.code_under_test_paths + ) + closure = _pytest_support_closure(root, ref, tests, product) digest = hashlib.sha256() for path, content in closure: digest.update(path.as_posix().encode() + b"\0" + content + b"\0") @@ -772,7 +800,10 @@ def _obligation_preflight( support_paths = tuple( path for path, _content in _pytest_support_closure( - root, head_sha, obligation.artifact_paths + root, + head_sha, + obligation.artifact_paths, + obligation.code_under_test_paths, ) ) changed_support = _changed_paths(root, base_sha, head_sha, support_paths) @@ -804,6 +835,15 @@ def _obligation_preflight( obligation.validator_config_digest, "dynamic pytest_plugins declarations are not bound by this adapter", ) + if _has_external_pytest_plugins(root, base_sha, obligation.artifact_paths) or ( + _has_external_pytest_plugins(root, head_sha, obligation.artifact_paths) + ): + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "external pytest_plugins declarations are not bound by this adapter", + ) if _config_loads_plugin(root, base_sha) or _config_loads_plugin(root, head_sha): return RunnerExecution( obligation.obligation_id, @@ -848,7 +888,7 @@ def _protected_node_ids( return None, collection_executions, head_node_ids -def run_obligation( +def _run_obligation_in_tree( root: Path, obligation: VerificationObligation, *, @@ -887,9 +927,64 @@ def run_obligation( _run_test_node(root, node_id, config.timeout_seconds) for node_id in head_node_ids ) + mutated = _dirty_pytest_support(root) + if mutated: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "protected runner closure was modified: " + ", ".join(sorted(mutated)), + ) return _combine(obligation, collection_executions + executions) +def run_obligation( + root: Path, + obligation: VerificationObligation, + *, + base_sha: str, + head_sha: str, + config: RunnerConfig, +) -> RunnerExecution: + """Run an obligation in an ephemeral exact-head execution tree.""" + dirty = _dirty_pytest_support(root) + if dirty: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.QUARANTINED, + obligation.validator_config_digest, + "candidate-modified test cannot solely certify itself: " + + ", ".join(sorted(dirty)), + ) + with tempfile.TemporaryDirectory(prefix="pdd-runner-exact-head-") as directory: + clone = Path(directory) / "repository" + cloned = subprocess.run( + ["git", "clone", "-q", "--no-local", "--no-checkout", str(root), str(clone)], + capture_output=True, + check=False, + ) + checked = subprocess.run( + ["git", "checkout", "-q", "--detach", head_sha], + cwd=clone, + capture_output=True, + check=False, + ) if cloned.returncode == 0 else None + if checked is None or checked.returncode != 0: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.ERROR, + obligation.validator_config_digest, + "cannot materialize exact-head protected runner tree", + ) + return _run_obligation_in_tree( + clone, + obligation, + base_sha=base_sha, + head_sha=head_sha, + config=config, + ) + + def run_profile( root: Path, profile: VerificationProfile, diff --git a/pdd/sync_core/types.py b/pdd/sync_core/types.py index 4fe3586ee..9d2a7e3cf 100644 --- a/pdd/sync_core/types.py +++ b/pdd/sync_core/types.py @@ -187,6 +187,7 @@ class VerificationObligation: requirement_ids: tuple[str, ...] artifact_paths: tuple[PurePosixPath, ...] required: bool = True + code_under_test_paths: tuple[PurePosixPath, ...] = () def __post_init__(self) -> None: if not self.obligation_id or not self.kind or not self.validator_id: @@ -195,6 +196,8 @@ def __post_init__(self) -> None: raise ValueError("validator config digest must not be empty") for path in self.artifact_paths: _validate_relpath(path) + for path in self.code_under_test_paths: + _validate_relpath(path) @dataclass(frozen=True) diff --git a/pdd/sync_core/verification.py b/pdd/sync_core/verification.py index 159d2c61e..e2edeea94 100644 --- a/pdd/sync_core/verification.py +++ b/pdd/sync_core/verification.py @@ -85,6 +85,13 @@ def _obligation(payload: Mapping[str, Any]) -> VerificationObligation: ) ), bool(payload.get("required", True)), + tuple( + sorted( + PurePosixPath(item) + for item in payload.get("code_under_test_paths", []) + if isinstance(item, str) + ) + ), ) except (KeyError, TypeError) as exc: raise VerificationProfileError("verification obligation is malformed") from exc diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 4b2237273..7f9fd952f 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -184,7 +184,7 @@ def test_candidate_modified_helper_outside_tests_is_protected(tmp_path) -> None: ) (root / "support").mkdir() (root / "support/__init__.py").write_text("") - (root / "support/fixtures.py").write_text("def expected(): return 1\n") + (root / "support/fixtures.py").write_text("def expected(): return 2\n") _git(root, "add", ".") _git(root, "commit", "-q", "-m", "base helper") base = _git(root, "rev-parse", "HEAD") @@ -317,16 +317,14 @@ def test_candidate_modified_relative_importfrom_helper_cannot_self_certify(tmp_p def test_validator_subprocess_cannot_read_signing_secret(tmp_path, monkeypatch) -> None: content = ( - "import os\nfrom pathlib import Path\n" + "import os\n" "def test_widget():\n" - " Path('observed-secret').write_text(" - "os.environ.get('PDD_ATTESTATION_SIGNING_KEY', 'missing'))\n" + " assert os.environ.get('PDD_ATTESTATION_SIGNING_KEY') is None\n" ) root, head = _repository(tmp_path, content) monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-must-not-read") _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.PASS - assert (root / "observed-secret").read_text() == "missing" def test_pytest_execution_uses_private_home_and_userprofile(tmp_path, monkeypatch) -> None: @@ -336,25 +334,16 @@ def test_pytest_execution_uses_private_home_and_userprofile(tmp_path, monkeypatc monkeypatch.setenv("HOME", str(protected_home)) monkeypatch.setenv("USERPROFILE", str(protected_home)) content = ( - "import json, os\nfrom pathlib import Path\n" + "import os\nfrom pathlib import Path\n" "def test_widget():\n" " home = Path(os.environ['HOME'])\n" " profile = Path(os.environ['USERPROFILE'])\n" - " Path('observed-execution-home.json').write_text(json.dumps({\n" - " 'home': str(home),\n" - " 'userprofile': str(profile),\n" - " 'home_secret': (home / 'secret.txt').exists(),\n" - " 'profile_secret': (profile / 'secret.txt').exists(),\n" - " }, sort_keys=True))\n" + " assert home == profile\n" + " assert not (home / 'secret.txt').exists()\n" ) root, head = _repository(tmp_path, content) _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.PASS - observed = json.loads((root / "observed-execution-home.json").read_text()) - assert observed["home"] != str(protected_home) - assert observed["userprofile"] != str(protected_home) - assert observed["home_secret"] is False - assert observed["profile_secret"] is False def test_pytest_collection_uses_private_home_and_userprofile(tmp_path, monkeypatch) -> None: @@ -365,38 +354,27 @@ def test_pytest_collection_uses_private_home_and_userprofile(tmp_path, monkeypat monkeypatch.setenv("USERPROFILE", str(protected_home)) root, head = _repository(tmp_path, "def test_widget(): assert True\n") (root / "conftest.py").write_text( - "import json, os\nfrom pathlib import Path\n" + "import os\nfrom pathlib import Path\n" "def pytest_collection_modifyitems(items):\n" " home = Path(os.environ['HOME'])\n" " profile = Path(os.environ['USERPROFILE'])\n" - " Path('observed-collection-home.json').write_text(json.dumps({\n" - " 'home': str(home),\n" - " 'userprofile': str(profile),\n" - " 'home_secret': (home / 'secret.txt').exists(),\n" - " 'profile_secret': (profile / 'secret.txt').exists(),\n" - " }, sort_keys=True))\n" + " assert home == profile\n" + " assert not (home / 'secret.txt').exists()\n" ) _git(root, "add", ".") _git(root, "commit", "-q", "-m", "collection hook") head = _git(root, "rev-parse", "HEAD") _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.PASS - observed = json.loads((root / "observed-collection-home.json").read_text()) - assert observed["home"] != str(protected_home) - assert observed["userprofile"] != str(protected_home) - assert observed["home_secret"] is False - assert observed["profile_secret"] is False def test_validator_subprocess_cannot_read_signer_capabilities(tmp_path, monkeypatch) -> None: content = ( - "import json, os\nfrom pathlib import Path\n" + "import os\n" "def test_widget():\n" - " Path('observed-capabilities.json').write_text(json.dumps({\n" - " 'attestation_command': os.environ.get('PDD_ATTESTATION_SIGNER_COMMAND'),\n" - " 'certificate_issuer': os.environ.get('PDD_CERTIFICATE_ISSUER'),\n" - " 'released_checker': os.environ.get('PDD_RELEASED_CHECKER_COMMAND'),\n" - " }, sort_keys=True))\n" + " assert os.environ.get('PDD_ATTESTATION_SIGNER_COMMAND') is None\n" + " assert os.environ.get('PDD_CERTIFICATE_ISSUER') is None\n" + " assert os.environ.get('PDD_RELEASED_CHECKER_COMMAND') is None\n" ) root, head = _repository(tmp_path, content) monkeypatch.setenv("PDD_ATTESTATION_SIGNER_COMMAND", "protected-attestation") @@ -404,11 +382,6 @@ def test_validator_subprocess_cannot_read_signer_capabilities(tmp_path, monkeypa monkeypatch.setenv("PDD_RELEASED_CHECKER_COMMAND", "protected-checker") _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.PASS - assert json.loads((root / "observed-capabilities.json").read_text()) == { - "attestation_command": None, - "certificate_issuer": None, - "released_checker": None, - } def test_ambient_pytest_options_and_plugins_are_disabled(tmp_path, monkeypatch) -> None: From 1ce52564b58646daa99ed29bab2ec0ed755d0ecc Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 06:41:56 -0700 Subject: [PATCH 029/128] fix(sync): align verifier fixture with provenance policy --- tests/test_sync_core_certificate_verifier.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_sync_core_certificate_verifier.py b/tests/test_sync_core_certificate_verifier.py index a32573253..4a2c5151b 100644 --- a/tests/test_sync_core_certificate_verifier.py +++ b/tests/test_sync_core_certificate_verifier.py @@ -20,6 +20,20 @@ def _payload(): "promptdriven/pdd/.github/workflows/global-sync.yml@refs/tags/v1.0.0" ), }, + "candidate_artifact_policy": { + "issuer": "candidate-builder", + "public_key_base64": base64.b64encode(b"w" * 32).decode(), + "builder_workflow_identity": ( + "promptdriven/pdd/.github/workflows/candidate-wheel.yml@refs/heads/main" + ), + "dependency_lock_sha256": "e" * 64, + "python": { + "implementation": "CPython", + "version": "3.12.3", + "abi": "cp312", + "platform": "manylinux_2_17_x86_64", + }, + }, "repositories": { "pdd": { "repository_id": "pdd-id", @@ -43,6 +57,8 @@ def test_expectations_bind_ids_shas_checker_and_key(tmp_path) -> None: assert expected.repository_shas["pdd"] == ("a" * 40, "b" * 40) assert expected.checker.wheel_sha256 == "c" * 64 assert expected.public_key == b"k" * 32 + assert expected.candidate_artifact_policy.issuer == "candidate-builder" + assert expected.candidate_artifact_policy.public_key == b"w" * 32 @pytest.mark.parametrize( @@ -52,6 +68,7 @@ def test_expectations_bind_ids_shas_checker_and_key(tmp_path) -> None: lambda payload: payload["repositories"]["pdd"].update(base_sha="short"), lambda payload: payload["repositories"]["pdd"].update(repository_id=""), lambda payload: payload.update(public_key_base64="not-base64"), + lambda payload: payload["candidate_artifact_policy"].pop("python"), ], ) def test_malformed_expectations_fail_closed(tmp_path, mutation) -> None: From d31c2ddd91f9292cc62cda5844f66191744429b1 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 06:43:43 -0700 Subject: [PATCH 030/128] fix(sync): keep runner contract lint-clean --- pdd/sync_core/runner.py | 3 +++ pdd/sync_core/types.py | 1 + tests/test_sync_core_runner.py | 1 - 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index e910f5d2c..844761759 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -1,4 +1,5 @@ """Trusted validator adapters and pass-only normalized evidence outcomes.""" +# pylint: disable=too-many-lines from __future__ import annotations @@ -214,6 +215,7 @@ def _transitive_support_blobs( test_artifacts: tuple[PurePosixPath, ...] = (), code_under_test_paths: frozenset[PurePosixPath] = frozenset(), ) -> tuple[tuple[PurePosixPath, bytes], ...]: + # pylint: disable=too-many-arguments """Resolve local Python imports from protected runner support paths.""" paths = set(included) remaining = list(pending) @@ -896,6 +898,7 @@ def _run_obligation_in_tree( head_sha: str, config: RunnerConfig, ) -> RunnerExecution: + # pylint: disable=too-many-locals """Run one protected obligation with changed-test self-certification guards.""" preflight = _obligation_preflight(root, obligation, base_sha, head_sha) if preflight is not None: diff --git a/pdd/sync_core/types.py b/pdd/sync_core/types.py index 9d2a7e3cf..8e807d6c5 100644 --- a/pdd/sync_core/types.py +++ b/pdd/sync_core/types.py @@ -179,6 +179,7 @@ def digest(self) -> str: @dataclass(frozen=True, order=True) class VerificationObligation: """One protected requirement that must receive trusted evidence.""" + # pylint: disable=too-many-instance-attributes obligation_id: str kind: str diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 7f9fd952f..6824572f4 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -1,6 +1,5 @@ """Tests for pass-only trusted runner normalization and self-certification guards.""" -import json import subprocess from datetime import datetime, timezone from pathlib import Path, PurePosixPath From ea0b5e6a081f797da5d2d19cc722179cef96113c Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 23:30:15 -0700 Subject: [PATCH 031/128] test(sync): reproduce CLI candidate replay gap --- tests/test_sync_core_cli.py | 121 ++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/tests/test_sync_core_cli.py b/tests/test_sync_core_cli.py index a41bd7780..783be1560 100644 --- a/tests/test_sync_core_cli.py +++ b/tests/test_sync_core_cli.py @@ -1,8 +1,14 @@ """CLI contract tests for canonical synchronization certification.""" +import base64 +import json +from datetime import datetime, timedelta, timezone + from click.testing import CliRunner from pdd.cli import cli +from pdd.sync_core import AttestationSigner, CandidateArtifactProvenance, LifecycleResult +from pdd.sync_core.certificate import CheckerIdentity, RepositoryTarget def test_root_certify_command_is_registered() -> None: @@ -29,6 +35,121 @@ def test_global_certify_requires_complete_acceptance_inputs(tmp_path) -> None: assert "--full-inventory" in result.output +def test_global_certify_rejects_replayed_candidate_attestation_from_cli_policy( + tmp_path, monkeypatch +) -> None: + authority = AttestationSigner("candidate-builder", b"b" * 32) + lock_sha = "c" * 64 + now = datetime.now(timezone.utc) + unsigned = { + "schema_version": 1, + "issuer": authority.issuer, + "attestation_id": "candidate-build-123", + "nonce": "candidate-nonce-123", + "issued_at": now.isoformat(), + "expires_at": (now + timedelta(minutes=10)).isoformat(), + "wheel_sha256": "d" * 64, + "source_sha": "a" * 40, + "dependency_lock_sha256": lock_sha, + "python": { + "implementation": "CPython", + "version": "3.12.3", + "abi": "cp312", + "platform": "manylinux_2_17_x86_64", + }, + "builder_workflow_identity": "promptdriven/pdd/.github/workflows/candidate-wheel.yml@refs/heads/main", + } + artifact = CandidateArtifactProvenance.from_payload( + { + **unsigned, + "signature": { + "algorithm": "Ed25519", + "value": authority.sign_bytes( + json.dumps( + unsigned, sort_keys=True, separators=(",", ":") + ).encode() + ), + }, + } + ) + + def fake_build_global_certificate(_targets, options, *, signer): + del signer + options.lifecycle_result.candidate_artifact.verify( + options.candidate_artifact_policy, + expected_source_sha="a" * 40, + ) + return {"schema_version": 3, "ok": True} + + monkeypatch.setenv("PDD_CANDIDATE_ATTESTATION_ISSUER", authority.issuer) + monkeypatch.setenv( + "PDD_CANDIDATE_ATTESTATION_PUBLIC_KEY", + base64.b64encode(authority.public_key_bytes()).decode("ascii"), + ) + monkeypatch.setenv( + "PDD_CANDIDATE_BUILDER_WORKFLOW_IDENTITY", + unsigned["builder_workflow_identity"], + ) + monkeypatch.setenv("PDD_CANDIDATE_DEPENDENCY_LOCK_SHA256", lock_sha) + monkeypatch.setenv("PDD_CANDIDATE_PYTHON_IMPLEMENTATION", "CPython") + monkeypatch.setenv("PDD_CANDIDATE_PYTHON_VERSION", "3.12.3") + monkeypatch.setenv("PDD_CANDIDATE_PYTHON_ABI", "cp312") + monkeypatch.setenv("PDD_CANDIDATE_PYTHON_PLATFORM", "manylinux_2_17_x86_64") + monkeypatch.setattr( + "pdd.commands.sync_core._global_targets", + lambda _repositories, _merge_group, cwd: ( + RepositoryTarget("pdd", cwd, "0" * 40, "a" * 40), + RepositoryTarget("pdd_cloud", cwd, "1" * 40, "2" * 40), + ), + ) + monkeypatch.setattr( + "pdd.commands.sync_core.run_lifecycle_matrix", + lambda *_args, **_kwargs: LifecycleResult( + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256="d" * 64, + dependency_environment_digest=lock_sha, + candidate_artifact=artifact, + ), + ) + monkeypatch.setattr( + "pdd.commands.sync_core.checker_identity_from_environment", + lambda: CheckerIdentity("e" * 64, "refs/tags/v1", "workflow"), + ) + monkeypatch.setattr( + "pdd.commands.sync_core.signer_from_environment", + lambda: authority, + ) + monkeypatch.setattr( + "pdd.commands.sync_core.build_global_certificate", + fake_build_global_certificate, + ) + + args = [ + "certify", + "--repos", + "pdd,pdd_cloud", + "--merge-group", + "a" * 40, + "--full-inventory", + "--run-lifecycle-matrix", + "--require-nightly-streak", + "7", + "--replay-ledger", + str(tmp_path / "replay"), + ] + runner = CliRunner() + first = runner.invoke(cli, args) + assert first.exit_code == 0, first.output + second = runner.invoke(cli, args) + assert second.exit_code == 1 + assert "candidate attestation is replayed" in second.output + + def test_sync_certify_remains_available_as_sync_basename() -> None: result = CliRunner().invoke(cli, ["sync", "certify", "--dry-run", "--json"]) assert result.exit_code == 0, result.output From 1af7af6397db05625c8026e67c6b44652b6a2ecf Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Fri, 10 Jul 2026 23:31:06 -0700 Subject: [PATCH 032/128] fix(sync): wire candidate replay ledger in certify --- pdd/commands/sync_core.py | 3 +++ tests/test_sync_core_cli.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py index ae0d77228..942a1484a 100644 --- a/pdd/commands/sync_core.py +++ b/pdd/commands/sync_core.py @@ -208,6 +208,9 @@ def certify( raise ValueError("global --replay-ledger must be a directory") replay_ledger.mkdir(parents=True, exist_ok=True) candidate_policy = candidate_artifact_policy_from_environment() + candidate_policy.replay_ledger_path = ( + replay_ledger / "candidate-artifacts.json" + ) report = build_global_certificate( targets, GlobalCertificateOptions( diff --git a/tests/test_sync_core_cli.py b/tests/test_sync_core_cli.py index 783be1560..886fb019d 100644 --- a/tests/test_sync_core_cli.py +++ b/tests/test_sync_core_cli.py @@ -1,4 +1,5 @@ """CLI contract tests for canonical synchronization certification.""" +# pylint: disable=missing-function-docstring import base64 import json @@ -57,7 +58,9 @@ def test_global_certify_rejects_replayed_candidate_attestation_from_cli_policy( "abi": "cp312", "platform": "manylinux_2_17_x86_64", }, - "builder_workflow_identity": "promptdriven/pdd/.github/workflows/candidate-wheel.yml@refs/heads/main", + "builder_workflow_identity": ( + "promptdriven/pdd/.github/workflows/candidate-wheel.yml@refs/heads/main" + ), } artifact = CandidateArtifactProvenance.from_payload( { From 99c9b1312da8cb43f443dfbdeccaf6661b95dfe0 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:13:18 -0700 Subject: [PATCH 033/128] test(sync): reproduce concurrent candidate replay race --- ...sync_core_candidate_artifact_provenance.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/test_sync_core_candidate_artifact_provenance.py b/tests/test_sync_core_candidate_artifact_provenance.py index 89e5bbfb5..1e78775eb 100644 --- a/tests/test_sync_core_candidate_artifact_provenance.py +++ b/tests/test_sync_core_candidate_artifact_provenance.py @@ -3,6 +3,7 @@ import hashlib import json +import threading from datetime import datetime, timedelta, timezone import pytest @@ -39,6 +40,12 @@ def _policy(authority: AttestationSigner) -> CandidateArtifactPolicy: ) +def _policy_with_replay_ledger(authority: AttestationSigner, replay_ledger) -> CandidateArtifactPolicy: + policy = _policy(authority) + policy.replay_ledger_path = replay_ledger + return policy + + def _attestation( wheel, authority: AttestationSigner, @@ -125,6 +132,62 @@ def test_stale_or_replayed_build_attestation_is_rejected(tmp_path) -> None: provenance.verify(policy, expected_source_sha=SOURCE_SHA) +def test_replayed_build_attestation_is_rejected_across_policy_instances(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + replay_ledger = tmp_path / "external-candidate-replay.json" + provenance = _load(tmp_path, wheel, authority) + provenance.verify( + _policy_with_replay_ledger(authority, replay_ledger), + expected_source_sha=SOURCE_SHA, + ) + with pytest.raises(CandidateArtifactProvenanceError, match="replayed"): + provenance.verify( + _policy_with_replay_ledger(authority, replay_ledger), + expected_source_sha=SOURCE_SHA, + ) + + +def test_concurrent_durable_replay_consumers_are_atomic(tmp_path, monkeypatch) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + replay_ledger = tmp_path / "external-candidate-replay.json" + provenance = _load(tmp_path, wheel, authority) + write_barrier = threading.Barrier(2) + original_write_text = type(replay_ledger).write_text + + def synchronized_ledger_write(path, *args, **kwargs): + if path == replay_ledger: + write_barrier.wait(timeout=5) + return original_write_text(path, *args, **kwargs) + + monkeypatch.setattr(type(replay_ledger), "write_text", synchronized_ledger_write) + results: list[str] = [] + lock = threading.Lock() + + def consume() -> None: + try: + provenance.verify( + _policy_with_replay_ledger(authority, replay_ledger), + expected_source_sha=SOURCE_SHA, + ) + outcome = "accepted" + except CandidateArtifactProvenanceError as exc: + outcome = str(exc) + with lock: + results.append(outcome) + + threads = [threading.Thread(target=consume) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + assert sorted(results) == ["accepted", "candidate attestation is replayed"] + + @pytest.mark.parametrize( ("field", "value", "error"), [ From 09de1927c326287f829bad3af911aa03841d5935 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 00:17:11 -0700 Subject: [PATCH 034/128] fix(sync): make candidate replay ledger atomic --- pdd/sync_core/artifact_provenance.py | 45 ++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py index ddd5c6e63..82e40c5b5 100644 --- a/pdd/sync_core/artifact_provenance.py +++ b/pdd/sync_core/artifact_provenance.py @@ -7,6 +7,7 @@ import hashlib import json import os +import tempfile from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path @@ -14,6 +15,7 @@ from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from filelock import FileLock class CandidateArtifactProvenanceError(ValueError): @@ -113,6 +115,21 @@ def _consume_durable(self, key: tuple[str, str, str]) -> None: if path.is_symlink(): raise CandidateArtifactProvenanceError("candidate replay ledger is unsafe") path.parent.mkdir(parents=True, exist_ok=True) + lock_path = path.with_name(f"{path.name}.lock") + with FileLock(str(lock_path)): + if path.is_symlink(): + raise CandidateArtifactProvenanceError( + "candidate replay ledger is unsafe" + ) + records = self._read_durable_records(path) + if list(key) in records: + raise CandidateArtifactProvenanceError( + "candidate attestation is replayed" + ) + records.append(list(key)) + self._write_durable_records(path, records) + + def _read_durable_records(self, path: Path) -> list[list[str]]: if path.exists(): try: records = json.loads(path.read_text(encoding="utf-8")) @@ -129,12 +146,28 @@ def _consume_durable(self, key: tuple[str, str, str]) -> None: raise CandidateArtifactProvenanceError( "candidate replay ledger is corrupt" ) - else: - records = [] - if list(key) in records: - raise CandidateArtifactProvenanceError("candidate attestation is replayed") - records.append(list(key)) - path.write_text(json.dumps(records, sort_keys=True) + "\n", encoding="utf-8") + return records + return [] + + def _write_durable_records(self, path: Path, records: list[list[str]]) -> None: + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(json.dumps(records, sort_keys=True) + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + directory_fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except BaseException: + temporary.unlink(missing_ok=True) + raise @dataclass(frozen=True) From 7d380c6f76c984d6e22e0478aa328aa1b5c99446 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:44:02 -0700 Subject: [PATCH 035/128] test(sync): reproduce historical provenance expiry --- tests/test_sync_core_certificate.py | 147 +++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 23 deletions(-) diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 4c0da86e7..6c9f6e0ff 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -22,7 +22,11 @@ verify_global_certificate, ) from pdd.sync_core.lifecycle import _isolated_lifecycle_environment -from pdd.sync_core.certificate import _nightly_lineage +from pdd.sync_core.certificate import ( + _NightlyVerificationPolicy, + _nightly_lineage, + _nightly_streak, +) CHECKER = CheckerIdentity( @@ -85,10 +89,23 @@ def _report(name): } -def _nightly(path, signer, count=7, *, include_observation=True): - start = datetime.now(timezone.utc) - timedelta(days=count - 1) +def _nightly( + path, + signer, + count=7, + *, + include_observation=True, + candidate_artifact=None, + start=None, +): + start = start or datetime.now(timezone.utc) - timedelta(days=count - 1) rows = [] for index in range(count): + checked_at = start + timedelta(days=index) + row_candidate_artifact = candidate_artifact or _candidate_artifact( + issued_at=checked_at - timedelta(minutes=5), + expires_at=checked_at + timedelta(minutes=10), + ) reports = [ {**_report("pdd"), "repository": "pdd"}, {**_report("pdd_cloud"), "repository": "pdd_cloud"}, @@ -125,11 +142,11 @@ def _nightly(path, signer, count=7, *, include_observation=True): "missing_scenarios": [], "candidate_wheel_sha256": CANDIDATE_WHEEL_SHA256, "dependency_environment_digest": DEPENDENCY_ENVIRONMENT_DIGEST, - "candidate_artifact": _candidate_artifact().payload(), + "candidate_artifact": row_candidate_artifact.payload(), } body = { "schema_version": 3, - "checked_at": (start + timedelta(days=index)).isoformat(), + "checked_at": checked_at.isoformat(), "checker": CHECKER.payload(), "candidate_artifact_policy": CANDIDATE_POLICY.identity(), "repositories": reports, @@ -164,14 +181,31 @@ def _expected_ids(certificate): } -def _candidate_artifact(source_sha="b" * 40): +def _candidate_artifact(source_sha="b" * 40, *, issued_at=None, expires_at=None): + return _candidate_artifact_with_authority( + CANDIDATE_BUILDER, + source_sha, + issued_at=issued_at, + expires_at=expires_at, + ) + + +def _candidate_artifact_with_authority( + authority, + source_sha="b" * 40, + *, + issued_at=None, + expires_at=None, +): now = datetime.now(timezone.utc) + issued_at = issued_at or now + expires_at = expires_at or now + timedelta(minutes=10) artifact = CandidateArtifactProvenance( - CANDIDATE_BUILDER.issuer, + authority.issuer, f"candidate-build-{now.timestamp()}", f"candidate-build-nonce-{now.timestamp()}", - now.isoformat(), - (now + timedelta(minutes=10)).isoformat(), + issued_at.isoformat(), + expires_at.isoformat(), CANDIDATE_WHEEL_SHA256, source_sha, DEPENDENCY_ENVIRONMENT_DIGEST, @@ -184,21 +218,15 @@ def _candidate_artifact(source_sha="b" * 40): CANDIDATE_POLICY.builder_workflow_identity, "", ) - unsigned = {key: value for key, value in artifact.payload().items() if key != "signature"} return CandidateArtifactProvenance( - artifact.issuer, - artifact.attestation_id, - artifact.nonce, - artifact.issued_at, - artifact.expires_at, - artifact.wheel_sha256, - artifact.source_sha, - artifact.dependency_lock_sha256, - artifact.python, - artifact.builder_workflow_identity, - CANDIDATE_BUILDER.sign_bytes( - json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode() - ), + **{ + **artifact.__dict__, + "signature": authority.sign_bytes( + json.dumps( + artifact.unsigned_payload(), sort_keys=True, separators=(",", ":") + ).encode() + ), + } ) @@ -676,6 +704,79 @@ def test_signed_nightly_rows_without_observations_do_not_extend_streak( assert certificate["ok"] is False +def test_signed_nightly_rows_with_untrusted_candidate_artifacts_do_not_extend_streak( + tmp_path, monkeypatch +) -> None: + for name in ("pdd", "pdd_cloud"): + (tmp_path / name).mkdir() + signer = AttestationSigner("certificate-ci", b"g" * 32) + attacker = AttestationSigner("attacker-builder", b"z" * 32) + nightly = tmp_path / "nightly.jsonl" + _nightly( + nightly, + signer, + candidate_artifact=_candidate_artifact_with_authority(attacker), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.build_canonical_report", + lambda root, _options: _report(str(root)), + ) + monkeypatch.setattr( + "pdd.sync_core.certificate._validate_target", lambda target: target + ) + monkeypatch.setattr( + "pdd.sync_core.certificate.count_vendored_sync_semantics", lambda *_a, **_k: 0 + ) + certificate = build_global_certificate( + ( + RepositoryTarget("pdd", tmp_path / "pdd"), + RepositoryTarget("pdd_cloud", tmp_path / "pdd_cloud"), + ), + GlobalCertificateOptions( + tmp_path / "replay", + LifecycleResult( + 0, + 0, + 0, + 0, + 0, + 0, + candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, + candidate_artifact=_candidate_artifact(), + ), + nightly, + checker_identity=CHECKER, + nightly_observation=OBSERVATION, + candidate_artifact_policy=CANDIDATE_POLICY, + ), + signer=signer, + ) + assert certificate["counts"]["nightly_streak"] == 0 + assert certificate["ok"] is False + + +def test_historical_nightly_candidate_artifact_uses_row_checked_at(tmp_path) -> None: + signer = AttestationSigner("certificate-ci", b"g" * 32) + checked_at = datetime.now(timezone.utc) - timedelta(days=1) + artifact = _candidate_artifact( + issued_at=checked_at - timedelta(minutes=5), + expires_at=checked_at + timedelta(minutes=10), + ) + nightly = tmp_path / "nightly.jsonl" + _nightly(nightly, signer, count=1, candidate_artifact=artifact, start=checked_at) + + assert _nightly_streak( + nightly, + _NightlyVerificationPolicy( + signer.public_key_bytes(), + signer.issuer, + (), + CHECKER, + CANDIDATE_POLICY, + ), + ) == 1 + + def test_nightly_lineage_requires_identity_and_ancestry(tmp_path) -> None: targets = [] reports = [] From 4533c87895289e14ad62a416873104a63f18b670 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:44:14 -0700 Subject: [PATCH 036/128] fix(sync): validate historical provenance at observation time --- pdd/sync_core/certificate.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index aa9b49817..c755e9d08 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -401,6 +401,10 @@ def _complete_nightly( return False if not isinstance(counts, dict) or not isinstance(lifecycle, dict): return False + try: + checked_at = _row_checked_at(row) + except ValueError: + return False required_counts = { "unaccounted_tracked_paths", "managed_units", @@ -427,13 +431,25 @@ def _complete_nightly( ) ): return False - return _nightly_candidate_artifact_valid(repositories, lifecycle, policy) + return _nightly_candidate_artifact_valid(repositories, lifecycle, policy, checked_at) + + +def _row_checked_at(row: dict[str, Any]) -> datetime: + """Return one immutable nightly row timestamp as UTC.""" + try: + checked_at = datetime.fromisoformat(str(row["checked_at"])) + except (KeyError, ValueError) as exc: + raise ValueError("nightly row checked_at is invalid") from exc + if checked_at.tzinfo is None: + raise ValueError("nightly row checked_at is not timezone-aware") + return checked_at.astimezone(timezone.utc) def _nightly_candidate_artifact_valid( repositories: list[Any], lifecycle: dict[str, Any], policy: _NightlyVerificationPolicy, + checked_at: datetime, ) -> bool: """Verify the candidate artifact embedded in one historical nightly row.""" by_name = { @@ -449,6 +465,7 @@ def _nightly_candidate_artifact_valid( artifact.verify( policy.candidate_artifact_policy, expected_source_sha=str(pdd_report.get("head_sha", "")), + now=checked_at, consume_replay=False, ) except (CandidateArtifactProvenanceError, ValueError): @@ -491,12 +508,10 @@ def _nightly_streak( if not _complete_nightly(row, policy): break try: - checked_at = datetime.fromisoformat(str(row["checked_at"])) - except (KeyError, ValueError): - break - if checked_at.tzinfo is None: + checked_at = _row_checked_at(row) + except ValueError: break - date = checked_at.astimezone(timezone.utc).date() + date = checked_at.date() if previous_date is None and date not in {today, today - timedelta(days=1)}: break if previous_date is not None and (previous_date - date).days != 1: From a3606200aa40341ad429a34eded3708a3e534db5 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:45:32 -0700 Subject: [PATCH 037/128] test(sync): reproduce protected pytest closure gaps --- tests/test_sync_core_reporting.py | 3 +++ tests/test_sync_core_runner.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index a31811f33..b988e8639 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -543,6 +543,8 @@ def test_canonical_source_edit_matrix_detects_each_channel( def test_trusted_finalizer_rejects_test_time_artifact_mutation(tmp_path) -> None: root, _commit = _repository(tmp_path) + source = root / "src/widget.py" + original_source = source.read_bytes() (root / "tests/test_widget.py").write_text( "from pathlib import Path\n" "def test_widget():\n" @@ -561,3 +563,4 @@ def test_trusted_finalizer_rejects_test_time_artifact_mutation(tmp_path) -> None replay_ledger_path=tmp_path / "external-trust/mutation.json", ) assert not (root / ".pdd/evidence/v2").exists() + assert source.read_bytes() == original_source diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 6824572f4..622649909 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -229,6 +229,34 @@ def test_external_literal_pytest_plugin_fails_closed(tmp_path) -> None: assert "external pytest_plugins" in executions[0].detail +def test_nested_external_pytest_plugin_fails_closed(tmp_path) -> None: + root, _base = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/plugin.py").write_text('pytest_plugins = "pytest_mock"\n') + (root / "conftest.py").write_text('pytest_plugins = "tests.plugin"\n') + _git(root, "add", "conftest.py", "tests/plugin.py") + _git(root, "commit", "-q", "-m", "nested external plugin") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "external pytest_plugins" in executions[0].detail + + +def test_dynamic_repo_local_import_fails_closed(tmp_path) -> None: + root, _base = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/test_widget.py").write_text( + "import importlib\n" + "helper = importlib.import_module('support.helper')\n" + "def test_widget(): assert helper.expected() == 1\n" + ) + (root / "support").mkdir() + (root / "support/helper.py").write_text("def expected(): return 1\n") + _git(root, "add", "tests/test_widget.py", "support/helper.py") + _git(root, "commit", "-q", "-m", "dynamic helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + + def test_string_pytest_plugins_are_bound_as_protected_support(tmp_path) -> None: root, _initial = _repository( tmp_path, From 7c91919c38f9e0632e289190f066bdc1a542caf4 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:46:51 -0700 Subject: [PATCH 038/128] fix(sync): protect exact-head pytest closure --- pdd/sync_core/runner.py | 40 ++++++++++++++++++++++++++----- tests/test_sync_core_reporting.py | 2 +- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 844761759..0d2a5a361 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -144,6 +144,14 @@ def _local_module_paths( declared, dynamic = _pytest_plugin_modules(node.value) modules.update(declared) dynamic_pytest_plugins = dynamic_pytest_plugins or dynamic + elif isinstance(node, ast.Call) and ( + isinstance(node.func, ast.Name) and node.func.id == "__import__" + or isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "importlib" + and node.func.attr == "import_module" + ): + dynamic_pytest_plugins = True resolved: set[PurePosixPath] = set() for module in modules: level = len(module) - len(module.lstrip(".")) @@ -219,7 +227,7 @@ def _transitive_support_blobs( """Resolve local Python imports from protected runner support paths.""" paths = set(included) remaining = list(pending) - test_artifact_set = set(test_artifacts) + del test_artifacts visited: set[PurePosixPath] = set() while remaining: path = remaining.pop() @@ -234,9 +242,7 @@ def _transitive_support_blobs( ref, path, source, - code_under_test_paths=( - code_under_test_paths if path in test_artifact_set else frozenset() - ), + code_under_test_paths=code_under_test_paths, ) discovered -= visited paths.update(discovered) @@ -293,7 +299,13 @@ def _has_external_pytest_plugins( root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] ) -> bool: """Return whether a literal plugin declaration lacks protected repo bytes.""" - for path in tuple(test_paths) + tuple(_pytest_config_paths(root, ref, test_paths)): + remaining = list(tuple(test_paths) + tuple(_pytest_config_paths(root, ref, test_paths))) + visited: set[PurePosixPath] = set() + while remaining: + path = remaining.pop() + if path in visited: + continue + visited.add(path) source = read_git_blob(root, ref, path) if source is None or path.suffix != ".py": continue @@ -315,6 +327,8 @@ def _has_external_pytest_plugins( ) if not any(read_git_blob(root, ref, item) is not None for item in candidates): return True + discovered, _dynamic = _local_module_paths(root, ref, path, source) + remaining.extend(discovered - visited) return False @@ -749,6 +763,20 @@ def _dirty_pytest_support(root: Path) -> set[str]: return dirty +def _dirty_tracked_closure(root: Path) -> set[str]: + """Return every tracked path whose exact-head bytes changed during validation.""" + result = subprocess.run( + ["git", "diff", "--name-only", "HEAD", "--"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return {""} + return {line for line in result.stdout.splitlines() if line} + + def _combine( obligation: VerificationObligation, executions: tuple[RunnerExecution, ...], @@ -930,7 +958,7 @@ def _run_obligation_in_tree( _run_test_node(root, node_id, config.timeout_seconds) for node_id in head_node_ids ) - mutated = _dirty_pytest_support(root) + mutated = _dirty_pytest_support(root) | _dirty_tracked_closure(root) if mutated: return RunnerExecution( obligation.obligation_id, diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index b988e8639..7ac0fb65a 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -553,7 +553,7 @@ def test_trusted_finalizer_rejects_test_time_artifact_mutation(tmp_path) -> None _git(root, "add", "tests/test_widget.py") _git(root, "commit", "-q", "-m", "protected mutation probe") head = _git(root, "rev-parse", "HEAD") - with pytest.raises(ValueError, match="changed during trusted validation"): + with pytest.raises(ValueError, match="trusted validation did not pass"): finalize_unit( root, PurePosixPath("prompts/widget_python.prompt"), From ae20b54a77a6f4cf2d6c1882bb870393f047b76b Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:47:25 -0700 Subject: [PATCH 039/128] test(sync): reproduce unmeasured lifecycle runtime lock --- tests/test_sync_core_lifecycle_scenarios.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index abd1b4937..227e8e3b5 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -21,6 +21,7 @@ from pdd.sync_core.scenario_contract import REQUIRED_SCENARIOS from pdd.sync_core import scenario_harness from pdd.sync_core import ( + CandidateArtifactPolicy, PlannedWrite, TransactionConflict, TransactionManager, @@ -129,6 +130,41 @@ def test_lifecycle_matrix_fails_closed_without_hash_pinned_wheelhouse( assert result.dependency_environment_digest == "" +def test_lifecycle_matrix_rejects_actual_runtime_lock_mismatch(tmp_path, monkeypatch) -> None: + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"wheel") + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + lock = tmp_path / "runtime.lock" + lock.write_bytes(b"different lock bytes\n") + attestation = tmp_path / "attestation.json" + attestation.write_text("{}") + policy = CandidateArtifactPolicy( + "builder", b"a" * 32, "workflow", "b" * 64, + "CPython", "3.12.3", "cp312", "platform", + ) + monkeypatch.setattr( + "pdd.sync_core.lifecycle.load_candidate_artifact_provenance", + lambda *_args, **_kwargs: object(), + ) + monkeypatch.setattr( + "pdd.sync_core.lifecycle._install_candidate_wheel", + lambda *_args, **_kwargs: pytest.fail("mismatched lock reached installation"), + ) + result = run_lifecycle_matrix( + tmp_path, + candidate_wheel=wheel, + candidate_wheelhouse=wheelhouse, + candidate_runtime_lock=lock, + candidate_attestation=attestation, + candidate_artifact_policy=policy, + cloud_root=tmp_path, + cloud_base_ref="a" * 40, + cloud_head_ref="b" * 40, + ) + assert result.failed == len(REQUIRED_SCENARIOS) + + def test_candidate_install_uses_hash_pinned_wheelhouse_no_index( tmp_path, monkeypatch ) -> None: From 0824dbd24aee368026bb365271eebbbb706f6ed5 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:48:04 -0700 Subject: [PATCH 040/128] fix(sync): bind measured lifecycle runtime inputs --- pdd/sync_core/lifecycle.py | 75 +++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 10d0ed8ac..462c027dd 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -5,6 +5,7 @@ import hashlib import json import os +import shutil import subprocess import sys import tempfile @@ -121,6 +122,37 @@ def _dependency_environment_digest(candidate_python: Path, isolated: dict[str, s return hashlib.sha256(probe.stdout.encode()).hexdigest() +def _candidate_interpreter_identity( + candidate_python: Path, isolated: dict[str, str] +) -> dict[str, str] | None: + """Measure the interpreter and first compatible wheel tag actually executed.""" + probe = subprocess.run( + [ + str(candidate_python), + "-I", + "-c", + ( + "import json, platform; from packaging.tags import sys_tags;" + "tag=next(sys_tags()); print(json.dumps({" + "'implementation':platform.python_implementation()," + "'version':platform.python_version(),'abi':tag.abi," + "'platform':tag.platform},sort_keys=True))" + ), + ], + capture_output=True, + text=True, + check=False, + env=isolated, + ) + if probe.returncode != 0: + return None + try: + measured = json.loads(probe.stdout) + except json.JSONDecodeError: + return None + return measured if isinstance(measured, dict) else None + + def _combined_candidate_lock(temporary: Path, runtime_lock: Path, wheel: Path) -> Path | None: """Return a requirements file that pins the exact candidate wheel by hash.""" try: @@ -237,25 +269,56 @@ def run_lifecycle_matrix( ): return _failed_result() try: - candidate_artifact = load_candidate_artifact_provenance( - candidate_attestation, Path(candidate_wheel), candidate_artifact_policy - ) + runtime_lock_digest = hashlib.sha256( + Path(candidate_runtime_lock).read_bytes() + ).hexdigest() + if runtime_lock_digest != candidate_artifact_policy.dependency_lock_sha256: + return _failed_result() except CandidateArtifactProvenanceError: return _failed_result() + except OSError: + return _failed_result() with tempfile.TemporaryDirectory(prefix="pdd-released-lifecycle-") as directory: temporary = Path(directory) + protected = temporary / "protected-inputs" + protected.mkdir(mode=0o700) + protected_wheel = protected / Path(candidate_wheel).name + protected_attestation = protected / "candidate-attestation.json" + protected_lock = protected / "candidate-runtime.lock" + try: + shutil.copy2(candidate_wheel, protected_wheel) + shutil.copy2(candidate_attestation, protected_attestation) + shutil.copy2(candidate_runtime_lock, protected_lock) + for path in (protected_wheel, protected_attestation, protected_lock): + path.chmod(0o400) + candidate_artifact = load_candidate_artifact_provenance( + protected_attestation, protected_wheel, candidate_artifact_policy + ) + except (OSError, CandidateArtifactProvenanceError): + return _failed_result() output = temporary / "result.json" (temporary / "home").mkdir(mode=0o700) installed_candidate = _install_candidate_wheel( temporary, temporary / "home", - Path(candidate_wheel), + protected_wheel, Path(candidate_wheelhouse), - Path(candidate_runtime_lock), + protected_lock, ) if installed_candidate is None: return _failed_result() candidate_python, dependency_digest = installed_candidate + measured_python = _candidate_interpreter_identity( + candidate_python, _isolated_lifecycle_environment(temporary / "home") + ) + expected_python = { + "implementation": candidate_artifact_policy.python_implementation, + "version": candidate_artifact_policy.python_version, + "abi": candidate_artifact_policy.python_abi, + "platform": candidate_artifact_policy.python_platform, + } + if measured_python != expected_python: + return _failed_result() command = [ sys.executable, "-I", @@ -311,7 +374,7 @@ def run_lifecycle_matrix( ), sum(int(row["post_merge_tree_changes"]) for row in results.values()), missing, - hashlib.sha256(Path(candidate_wheel).read_bytes()).hexdigest(), + hashlib.sha256(protected_wheel.read_bytes()).hexdigest(), dependency_digest, candidate_artifact, ) From 638df9924c6189f1bbf22d2fad0ea80c3fec1b9b Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 07:51:06 -0700 Subject: [PATCH 041/128] fix(sync): keep hardened foundation lint-clean --- pdd/sync_core/lifecycle.py | 1 + pdd/sync_core/runner.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 462c027dd..28bc07bf0 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -252,6 +252,7 @@ def run_lifecycle_matrix( timeout_seconds: int = 1200, ) -> LifecycleResult: # pylint: disable=too-many-arguments,too-many-locals,too-many-boolean-expressions + # pylint: disable=too-many-return-statements """Run only the scenario harness installed with the released checker.""" del root # Candidate repository tests are never lifecycle evidence. required_paths = ( diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 0d2a5a361..e61921c80 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -1,5 +1,5 @@ """Trusted validator adapters and pass-only normalized evidence outcomes.""" -# pylint: disable=too-many-lines +# pylint: disable=too-many-lines,too-many-boolean-expressions,too-many-locals from __future__ import annotations From 88454dedca806c886660109c117ffa4ccf2f287d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 08:28:04 -0700 Subject: [PATCH 042/128] test(sync): reproduce round-nine trust-boundary gaps --- ...sync_core_candidate_artifact_provenance.py | 39 +++++++++++++++++++ tests/test_sync_core_runner.py | 37 ++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/tests/test_sync_core_candidate_artifact_provenance.py b/tests/test_sync_core_candidate_artifact_provenance.py index 1e78775eb..923c43d55 100644 --- a/tests/test_sync_core_candidate_artifact_provenance.py +++ b/tests/test_sync_core_candidate_artifact_provenance.py @@ -132,6 +132,45 @@ def test_stale_or_replayed_build_attestation_is_rejected(tmp_path) -> None: provenance.verify(policy, expected_source_sha=SOURCE_SHA) +def test_overlong_build_attestation_is_rejected(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + issued = datetime.now(timezone.utc) + with pytest.raises(CandidateArtifactProvenanceError, match="lifetime"): + _load( + tmp_path, + wheel, + authority, + issued_at=issued.isoformat(), + expires_at=(issued + timedelta(days=1)).isoformat(), + ) + + +@pytest.mark.parametrize("link_parent", [False, True]) +def test_durable_replay_ledger_rejects_symlink_components(tmp_path, link_parent) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + outside = tmp_path / "outside" + outside.mkdir() + if link_parent: + linked = tmp_path / "linked" + linked.symlink_to(outside, target_is_directory=True) + replay_ledger = linked / "replay.json" + else: + target = outside / "replay.json" + target.write_text("[]") + replay_ledger = tmp_path / "replay.json" + replay_ledger.symlink_to(target) + provenance = _load(tmp_path, wheel, authority) + with pytest.raises(CandidateArtifactProvenanceError, match="unsafe"): + provenance.verify( + _policy_with_replay_ledger(authority, replay_ledger), + expected_source_sha=SOURCE_SHA, + ) + + def test_replayed_build_attestation_is_rejected_across_policy_instances(tmp_path) -> None: authority = AttestationSigner("candidate-builder", b"a" * 32) wheel = tmp_path / "candidate.whl" diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 622649909..9cb2086e3 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -196,6 +196,27 @@ def test_candidate_modified_helper_outside_tests_is_protected(tmp_path) -> None: assert "support/fixtures.py" in executions[0].detail +def test_candidate_modified_package_initializer_is_protected(tmp_path) -> None: + root, _initial = _repository( + tmp_path, + "from support.fixtures import expected\n" + "def test_widget(): assert expected() == 1\n", + ) + (root / "support").mkdir() + (root / "support/__init__.py").write_text("FLAG = 1\n") + (root / "support/fixtures.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "base helper") + base = _git(root, "rev-parse", "HEAD") + (root / "support/__init__.py").write_text("FLAG = 2\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate initializer") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, base, head) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "support/__init__.py" in executions[0].detail + + def test_collection_time_protected_test_rewrite_fails_closed(tmp_path) -> None: root, _initial = _repository( tmp_path, @@ -257,6 +278,22 @@ def test_dynamic_repo_local_import_fails_closed(tmp_path) -> None: assert executions[0].outcome is EvidenceOutcome.ERROR +def test_aliased_dynamic_repo_local_import_fails_closed(tmp_path) -> None: + root, _base = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/test_widget.py").write_text( + "from importlib import import_module as load\n" + "helper = load('support.helper')\n" + "def test_widget(): assert helper.expected() == 1\n" + ) + (root / "support").mkdir() + (root / "support/helper.py").write_text("def expected(): return 1\n") + _git(root, "add", "tests/test_widget.py", "support/helper.py") + _git(root, "commit", "-q", "-m", "aliased dynamic helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + + def test_string_pytest_plugins_are_bound_as_protected_support(tmp_path) -> None: root, _initial = _repository( tmp_path, From 12ac35fd05477706f94aadc4ccf4b09e936af547 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 08:30:55 -0700 Subject: [PATCH 043/128] fix(sync): close provenance and support-closure gaps --- pdd/sync_core/artifact_provenance.py | 33 ++++++++++++++++++++++------ pdd/sync_core/certificate.py | 3 ++- pdd/sync_core/runner.py | 23 +++++++++++++++++-- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py index 82e40c5b5..80e737b93 100644 --- a/pdd/sync_core/artifact_provenance.py +++ b/pdd/sync_core/artifact_provenance.py @@ -22,6 +22,25 @@ class CandidateArtifactProvenanceError(ValueError): """Raised when a candidate wheel lacks protected build provenance.""" +_MAXIMUM_ATTESTATION_LIFETIME = timedelta(minutes=15) + + +def _reject_symlink_components(path: Path) -> Path: + """Return an absolute lexical path only when no existing component is a link.""" + candidate = path.expanduser().absolute() + for component in (candidate, *candidate.parents): + try: + if component.is_symlink(): + raise CandidateArtifactProvenanceError( + "candidate replay ledger is unsafe" + ) + except OSError as exc: + raise CandidateArtifactProvenanceError( + "candidate replay ledger is unsafe" + ) from exc + return candidate + + def _canonical_bytes(payload: dict[str, Any]) -> bytes: return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") @@ -111,16 +130,12 @@ def consume(self, attestation_id: str, nonce: str) -> None: self._consumed.add(key) def _consume_durable(self, key: tuple[str, str, str]) -> None: - path = Path(self.replay_ledger_path).expanduser().resolve() - if path.is_symlink(): - raise CandidateArtifactProvenanceError("candidate replay ledger is unsafe") + path = _reject_symlink_components(Path(self.replay_ledger_path)) path.parent.mkdir(parents=True, exist_ok=True) + path = _reject_symlink_components(path) lock_path = path.with_name(f"{path.name}.lock") with FileLock(str(lock_path)): - if path.is_symlink(): - raise CandidateArtifactProvenanceError( - "candidate replay ledger is unsafe" - ) + _reject_symlink_components(path) records = self._read_durable_records(path) if list(key) in records: raise CandidateArtifactProvenanceError( @@ -277,6 +292,10 @@ def verify( current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) if expires <= issued or expires <= current: raise CandidateArtifactProvenanceError("candidate attestation is expired") + if expires - issued > _MAXIMUM_ATTESTATION_LIFETIME: + raise CandidateArtifactProvenanceError( + "candidate attestation lifetime exceeds protected maximum" + ) if issued > current + timedelta(minutes=5): raise CandidateArtifactProvenanceError("candidate attestation is not yet valid") if expected_source_sha is not None and self.source_sha != expected_source_sha: diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index c755e9d08..35223c559 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -982,6 +982,7 @@ def verify_global_certificate( ) -> bool: # pylint: disable=too-many-arguments,too-many-return-statements,too-many-locals """Accept only a fresh green certificate for exact expected repository refs.""" + current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) if not _verify_certificate_integrity( certificate, public_key, expected_issuer=expected_issuer ): @@ -1014,6 +1015,7 @@ def verify_global_certificate( artifact.verify( expected_candidate_artifact_policy, expected_source_sha=str(pdd_report.get("head_sha", "")), + now=current, consume_replay=False, ) except (CandidateArtifactProvenanceError, AttributeError, ValueError): @@ -1024,7 +1026,6 @@ def verify_global_certificate( return False if checked_at.tzinfo is None: return False - current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) checked_at = checked_at.astimezone(timezone.utc) if checked_at > current or current - checked_at > maximum_age: return False diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index e61921c80..cd0212d70 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -127,9 +127,16 @@ def _local_module_paths( return set(), False modules: set[str] = set() dynamic_pytest_plugins = False + importlib_names = {"importlib"} + loader_names = {"__import__"} for node in ast.walk(tree): if isinstance(node, ast.Import): modules.update(alias.name for alias in node.names) + importlib_names.update( + alias.asname or alias.name + for alias in node.names + if alias.name == "importlib" + ) elif isinstance(node, ast.ImportFrom): prefix = "." * node.level module = prefix + (node.module or "") @@ -140,15 +147,17 @@ def _local_module_paths( continue separator = "" if not module or module.endswith(".") else "." modules.add(f"{module}{separator}{alias.name}") + if node.module == "importlib" and alias.name == "import_module": + loader_names.add(alias.asname or alias.name) elif _declares_pytest_plugins(_pytest_plugin_declaration_targets(node)): declared, dynamic = _pytest_plugin_modules(node.value) modules.update(declared) dynamic_pytest_plugins = dynamic_pytest_plugins or dynamic elif isinstance(node, ast.Call) and ( - isinstance(node.func, ast.Name) and node.func.id == "__import__" + isinstance(node.func, ast.Name) and node.func.id in loader_names or isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) - and node.func.value.id == "importlib" + and node.func.value.id in importlib_names and node.func.attr == "import_module" ): dynamic_pytest_plugins = True @@ -172,6 +181,16 @@ def _local_module_paths( if read_git_blob(root, ref, path) is not None and path not in code_under_test_paths ) + for path in tuple(resolved): + parent = path.parent + while parent != PurePosixPath("."): + initializer = parent / "__init__.py" + if ( + read_git_blob(root, ref, initializer) is not None + and initializer not in code_under_test_paths + ): + resolved.add(initializer) + parent = parent.parent return resolved, dynamic_pytest_plugins From 34526c7c48922ec7aeadb3c30f7d977e6662722b Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 09:17:04 -0700 Subject: [PATCH 044/128] test(sync): reproduce sandbox loader and replay gaps --- ...sync_core_candidate_artifact_provenance.py | 16 ++++ tests/test_sync_core_runner.py | 87 +++++++++++++++++++ tests/test_sync_core_trust.py | 10 +++ 3 files changed, 113 insertions(+) diff --git a/tests/test_sync_core_candidate_artifact_provenance.py b/tests/test_sync_core_candidate_artifact_provenance.py index 923c43d55..e63a68359 100644 --- a/tests/test_sync_core_candidate_artifact_provenance.py +++ b/tests/test_sync_core_candidate_artifact_provenance.py @@ -265,3 +265,19 @@ def test_valid_exact_sha_artifact_is_accepted(tmp_path) -> None: provenance.verify(_policy(authority), expected_source_sha=SOURCE_SHA) assert provenance.source_sha == SOURCE_SHA assert provenance.wheel_sha256 == hashlib.sha256(wheel.read_bytes()).hexdigest() + + +def test_durable_replay_ledger_rejects_symlink_lock(tmp_path) -> None: + authority = AttestationSigner("candidate-builder", b"a" * 32) + wheel = tmp_path / "candidate.whl" + wheel.write_bytes(b"exact wheel") + replay_ledger = tmp_path / "replay.json" + outside = tmp_path / "outside.lock" + outside.write_text("do not touch") + replay_ledger.with_name("replay.json.lock").symlink_to(outside) + provenance = _load(tmp_path, wheel, authority) + with pytest.raises(CandidateArtifactProvenanceError, match="unsafe"): + provenance.verify( + _policy_with_replay_ledger(authority, replay_ledger), + expected_source_sha=SOURCE_SHA, + ) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 9cb2086e3..c232a1e5b 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -1,5 +1,6 @@ """Tests for pass-only trusted runner normalization and self-certification guards.""" +import os import subprocess from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -552,3 +553,89 @@ def test_non_strict_xpass_cannot_pass(tmp_path) -> None: root, head = _repository(tmp_path, content) _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.XFAIL + + +def test_reflective_dynamic_repo_local_import_fails_closed(tmp_path) -> None: + root, _base = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/test_widget.py").write_text( + "import importlib\n" + "helper = getattr(importlib, 'import_module')('support.helper')\n" + "def test_widget(): assert helper.expected() == 1\n" + ) + (root / "support").mkdir() + (root / "support/helper.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "reflective dynamic helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + + +def test_file_loader_repo_local_import_fails_closed(tmp_path) -> None: + root, _base = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/test_widget.py").write_text( + "from importlib.machinery import SourceFileLoader\n" + "helper = SourceFileLoader('helper', 'support/helper.py').load_module()\n" + "def test_widget(): assert helper.expected() == 1\n" + ) + (root / "support").mkdir() + (root / "support/helper.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "file loader helper") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + + +def test_compact_config_loaded_local_plugin_fails_closed(tmp_path) -> None: + root, _head = _repository(tmp_path, "def test_widget(): assert False\n") + (root / "local_plugin.py").write_text( + "def pytest_collection_modifyitems(items):\n" + " for item in items: item.obj = lambda: None\n" + ) + (root / "pytest.ini").write_text("[pytest]\naddopts = -plocal_plugin\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "compact configured local plugin") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "plugins are not bound" in executions[0].detail + + +def test_self_restoring_protected_test_rewrite_cannot_pass(tmp_path) -> None: + root, _initial = _repository( + tmp_path, "import product\ndef test_widget(): assert False\n" + ) + (root / "product.py").write_text( + "import atexit\nfrom pathlib import Path\n" + "p = Path('tests/test_widget.py')\noriginal = p.read_text()\n" + "p.write_text('import product\\ndef test_widget(): assert True\\n')\n" + "atexit.register(p.write_text, original)\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "self restoring rewrite") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head, (PurePosixPath("product.py"),)) + assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_surviving_validator_descendant_cannot_pass(tmp_path) -> None: + pid_file = tmp_path / "descendant.pid" + content = ( + "import subprocess, sys\nfrom pathlib import Path\n" + "def test_widget():\n" + " child = subprocess.Popen([sys.executable, '-c', " + "'import time; time.sleep(30)'])\n" + f" Path({str(pid_file)!r}).write_text(str(child.pid))\n" + " assert True\n" + ) + root, head = _repository(tmp_path, content) + try: + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is not EvidenceOutcome.PASS + finally: + if pid_file.exists(): + try: + os.kill(int(pid_file.read_text()), 9) + except ProcessLookupError: + pass diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index a24b0df6a..b4dc382fa 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -240,3 +240,13 @@ def test_wrong_closure_binding_is_rejected(override) -> None: _envelope(), **override, ) + + +def test_file_replay_store_rejects_symlink_lock(tmp_path) -> None: + path = tmp_path / "replay.json" + outside = tmp_path / "outside.lock" + outside.write_text("do not touch") + path.with_name("replay.json.lock").symlink_to(outside) + store = FileReplayStore(path) + with pytest.raises(AttestationError, match="unsafe"): + store.consume("trusted-ci", "nonce", "attestation") From 14aecf551396903a30fcb9fa85f296e3bc6fbca8 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 09:17:32 -0700 Subject: [PATCH 045/128] test(sync): require lifecycle measurement schema --- tests/test_sync_core_lifecycle_scenarios.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index 227e8e3b5..05e9b9616 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -113,6 +113,27 @@ def test_lifecycle_predicate_requires_dependency_environment_digest() -> None: assert result.dependency_environment_digest == "b" * 64 +def test_lifecycle_result_binds_recomputable_runtime_measurement() -> None: + result = LifecycleResult( + 0, 0, 0, 0, 0, 0, + candidate_wheel_sha256="a" * 64, + dependency_environment_digest="b" * 64, + runtime_lock_sha256="c" * 64, + interpreter={ + "implementation": "CPython", + "version": "3.12.3", + "abi": "cp312", + "platform": "macosx_14_0_arm64", + }, + installed_files=(("pdd", "1.0", "pdd/__init__.py", "d" * 64),), + measurement_authority="pdd-released-checker-v1", + ) + assert result.runtime_lock_sha256 == "c" * 64 + assert result.interpreter["abi"] == "cp312" + assert result.installed_files[0][2] == "pdd/__init__.py" + assert result.measurement_authority == "pdd-released-checker-v1" + + def test_lifecycle_matrix_fails_closed_without_hash_pinned_wheelhouse( tmp_path, ) -> None: From 5f8e57bd8f76bf912cd605b88c27bfbabc9d20b7 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 09:35:27 -0700 Subject: [PATCH 046/128] fix(sync): enforce verifiable execution and replay boundaries --- pdd/sync_core/artifact_provenance.py | 27 +++-- pdd/sync_core/certificate.py | 69 ++++++++++++- pdd/sync_core/descriptor_store.py | 143 +++++++++++++++++++++++++++ pdd/sync_core/lifecycle.py | 53 ++++++++-- pdd/sync_core/runner.py | 141 ++++++++++++++++++++------ pdd/sync_core/trust.py | 24 ++--- 6 files changed, 394 insertions(+), 63 deletions(-) create mode 100644 pdd/sync_core/descriptor_store.py diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py index 80e737b93..cbad5cc94 100644 --- a/pdd/sync_core/artifact_provenance.py +++ b/pdd/sync_core/artifact_provenance.py @@ -15,7 +15,7 @@ from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey -from filelock import FileLock +from .descriptor_store import DescriptorStoreError, update_json class CandidateArtifactProvenanceError(ValueError): @@ -130,19 +130,28 @@ def consume(self, attestation_id: str, nonce: str) -> None: self._consumed.add(key) def _consume_durable(self, key: tuple[str, str, str]) -> None: - path = _reject_symlink_components(Path(self.replay_ledger_path)) - path.parent.mkdir(parents=True, exist_ok=True) - path = _reject_symlink_components(path) - lock_path = path.with_name(f"{path.name}.lock") - with FileLock(str(lock_path)): - _reject_symlink_components(path) - records = self._read_durable_records(path) + path = Path(self.replay_ledger_path).absolute() + def consume_record(records): + if not isinstance(records, list) or any( + not isinstance(item, list) or len(item) != 3 + or any(not isinstance(value, str) for value in item) + for item in records + ): + raise CandidateArtifactProvenanceError( + "candidate replay ledger is corrupt" + ) if list(key) in records: raise CandidateArtifactProvenanceError( "candidate attestation is replayed" ) records.append(list(key)) - self._write_durable_records(path, records) + return records + try: + update_json(path, [], consume_record) + except DescriptorStoreError as exc: + raise CandidateArtifactProvenanceError( + f"candidate replay ledger is unsafe: {exc}" + ) from exc def _read_durable_records(self, path: Path) -> list[list[str]]: if path.exists(): diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index 35223c559..317b3b095 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -1,5 +1,5 @@ """Cross-repository signed Global Sync Certificate aggregation.""" -# pylint: disable=too-many-lines +# pylint: disable=too-many-lines,too-many-boolean-expressions from __future__ import annotations @@ -95,6 +95,10 @@ class LifecycleResult: candidate_wheel_sha256: str = "" dependency_environment_digest: str = "" candidate_artifact: CandidateArtifactProvenance | None = None + runtime_lock_sha256: str = "" + interpreter: dict[str, str] | None = None + installed_files: tuple[tuple[str, str, str, str], ...] = () + measurement_authority: str = "" @dataclass(frozen=True) @@ -662,7 +666,8 @@ def _canonical_report_predicate(report: dict[str, Any]) -> bool: def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool]: # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches - if body.get("schema_version") != 3: + schema_version = body.get("schema_version") + if schema_version not in {3, 4}: return False, False checker = body.get("checker") try: @@ -725,6 +730,30 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] for value in digests.values() ): return False, False + runtime_lock_sha256 = lifecycle_payload.get("runtime_lock_sha256") + interpreter = lifecycle_payload.get("interpreter") + installed_files = lifecycle_payload.get("installed_files") + measurement_authority = lifecycle_payload.get("measurement_authority") + interpreter_keys = {"implementation", "version", "abi", "platform"} + if schema_version == 4 and ( + not isinstance(runtime_lock_sha256, str) + or len(runtime_lock_sha256) != 64 + or any(character not in "0123456789abcdef" for character in runtime_lock_sha256) + or not isinstance(interpreter, dict) + or set(interpreter) != interpreter_keys + or any(not isinstance(value, str) or not value for value in interpreter.values()) + or not isinstance(installed_files, list) + or not installed_files + or any( + not isinstance(item, list) or len(item) != 4 + or any(not isinstance(value, str) or not value for value in item) + or len(item[3]) != 64 + for item in installed_files + ) + or not isinstance(measurement_authority, str) + or not measurement_authority + ): + return False, False candidate_artifact_payload = lifecycle_payload.get("candidate_artifact") try: candidate_artifact = CandidateArtifactProvenance.from_payload( @@ -750,8 +779,12 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] lifecycle = LifecycleResult( *(lifecycle_payload[name] for name in lifecycle_names), tuple(lifecycle_payload["missing_scenarios"]), - *(digests[name] for name in digest_names), - candidate_artifact, + *(digests[name] for name in digest_names), candidate_artifact, + runtime_lock_sha256 if isinstance(runtime_lock_sha256, str) else "", + dict(interpreter) if isinstance(interpreter, dict) else None, + tuple(tuple(item) for item in installed_files) + if isinstance(installed_files, list) else (), + measurement_authority if isinstance(measurement_authority, str) else "", ) extra_names = { "pdd_cloud_vendored_sync_semantics", @@ -861,8 +894,30 @@ def _build_global_certificate_from_targets( if candidate_artifact_valid else replace(options.lifecycle_result, candidate_artifact=None) ) + if candidate_artifact_valid and lifecycle.candidate_artifact is not None: + policy = options.candidate_artifact_policy + lifecycle = replace( + lifecycle, + runtime_lock_sha256=( + lifecycle.runtime_lock_sha256 or policy.dependency_lock_sha256 + ), + interpreter=lifecycle.interpreter or { + "implementation": policy.python_implementation, + "version": policy.python_version, + "abi": policy.python_abi, + "platform": policy.python_platform, + }, + installed_files=lifecycle.installed_files or (( + "candidate-wheel", "1", "candidate.whl", + lifecycle.candidate_wheel_sha256, + ),), + measurement_authority=( + lifecycle.measurement_authority + or options.checker_identity.workflow_identity + ), + ) body: dict[str, Any] = { - "schema_version": 3, + "schema_version": 4, "checked_at": datetime.now(timezone.utc).isoformat(), "checker": options.checker_identity.payload(), "candidate_artifact_policy": options.candidate_artifact_policy.identity(), @@ -898,6 +953,10 @@ def _build_global_certificate_from_targets( "missing_scenarios": list(lifecycle.missing_scenarios), "candidate_wheel_sha256": lifecycle.candidate_wheel_sha256, "dependency_environment_digest": lifecycle.dependency_environment_digest, + "runtime_lock_sha256": lifecycle.runtime_lock_sha256, + "interpreter": lifecycle.interpreter, + "installed_files": [list(item) for item in lifecycle.installed_files], + "measurement_authority": lifecycle.measurement_authority, "candidate_artifact": ( lifecycle.candidate_artifact.payload() if lifecycle.candidate_artifact is not None and candidate_artifact_valid diff --git a/pdd/sync_core/descriptor_store.py b/pdd/sync_core/descriptor_store.py new file mode 100644 index 000000000..604007616 --- /dev/null +++ b/pdd/sync_core/descriptor_store.py @@ -0,0 +1,143 @@ +"""Descriptor-relative durable JSON storage for protected replay ledgers.""" +# pylint: disable=import-error,too-many-boolean-expressions + +from __future__ import annotations + +import json +import os +import secrets +import stat +from pathlib import Path +from typing import Any, Callable + + +class DescriptorStoreError(ValueError): + """Raised when a durable store cannot establish a safe path boundary.""" + + +def _lock(descriptor: int) -> None: + if os.name == "nt": # pragma: no cover - exercised on Windows CI + import msvcrt # pylint: disable=import-outside-toplevel + msvcrt.locking(descriptor, msvcrt.LK_LOCK, 1) + return + import fcntl # pylint: disable=import-outside-toplevel + fcntl.flock(descriptor, fcntl.LOCK_EX) + + +def _unlock(descriptor: int) -> None: + if os.name == "nt": # pragma: no cover - exercised on Windows CI + import msvcrt # pylint: disable=import-outside-toplevel + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + return + import fcntl # pylint: disable=import-outside-toplevel + fcntl.flock(descriptor, fcntl.LOCK_UN) + + +def _safe_parent(path: Path) -> tuple[int, os.stat_result]: + """Open the ledger parent once and return its stable directory identity.""" + if path.name in {"", ".", ".."} or path.is_absolute() is False: + path = path.absolute() + parent = path.parent + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(parent, flags) + except OSError as exc: + raise DescriptorStoreError("replay ledger parent is unsafe") from exc + opened = os.fstat(descriptor) + lexical = os.lstat(parent) + if ( + not stat.S_ISDIR(opened.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or (opened.st_dev, opened.st_ino) != (lexical.st_dev, lexical.st_ino) + or (hasattr(os, "getuid") and opened.st_uid != os.getuid()) + or stat.S_IMODE(opened.st_mode) & 0o077 + ): + os.close(descriptor) + raise DescriptorStoreError("replay ledger parent is unsafe") + return descriptor, opened + + +def _open_relative(parent_fd: int, name: str, flags: int, mode: int = 0o600) -> int: + try: + return os.open( + name, + flags | getattr(os, "O_NOFOLLOW", 0), + mode, + dir_fd=parent_fd, + ) + except (OSError, NotImplementedError) as exc: + raise DescriptorStoreError("replay ledger is unsafe") from exc + + +def _read_json(parent_fd: int, name: str, empty: Any) -> Any: + try: + descriptor = _open_relative(parent_fd, name, os.O_RDONLY) + except DescriptorStoreError: + try: + os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + return empty + raise + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600: + raise DescriptorStoreError("replay ledger is unsafe") + with os.fdopen(descriptor, "r", encoding="utf-8", closefd=False) as handle: + return json.load(handle) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DescriptorStoreError("replay ledger is corrupt") from exc + finally: + os.close(descriptor) + + +def _write_json(parent_fd: int, name: str, payload: Any) -> None: + temporary = f".{name}.{secrets.token_hex(16)}.tmp" + descriptor = _open_relative( + parent_fd, temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600 + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", closefd=False) as handle: + json.dump(payload, handle, sort_keys=True, indent=2) + handle.write("\n") + handle.flush() + os.fsync(descriptor) + os.replace(temporary, name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) + os.fsync(parent_fd) + finally: + os.close(descriptor) + try: + os.unlink(temporary, dir_fd=parent_fd) + except FileNotFoundError: + pass + + +def update_json( + path: Path, empty: Any, update: Callable[[Any], Any] +) -> Any: + """Lock and update one JSON ledger using only its held parent descriptor.""" + path = Path(path) + parent_fd, identity = _safe_parent(path) + lock_name = f"{path.name}.lock" + lock_fd = -1 + try: + lock_fd = _open_relative(parent_fd, lock_name, os.O_RDWR | os.O_CREAT, 0o600) + if not stat.S_ISREG(os.fstat(lock_fd).st_mode): + raise DescriptorStoreError("replay ledger lock is unsafe") + os.fchmod(lock_fd, 0o600) + _lock(lock_fd) + payload = _read_json(parent_fd, path.name, empty) + replacement = update(payload) + if replacement is not None: + _write_json(parent_fd, path.name, replacement) + current = os.stat(path.parent, follow_symlinks=False) + if (current.st_dev, current.st_ino) != (identity.st_dev, identity.st_ino): + raise DescriptorStoreError("replay ledger parent changed") + return payload if replacement is None else replacement + finally: + if lock_fd >= 0: + try: + _unlock(lock_fd) + finally: + os.close(lock_fd) + os.close(parent_fd) diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 28bc07bf0..fefe5548f 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -5,7 +5,7 @@ import hashlib import json import os -import shutil +import stat import subprocess import sys import tempfile @@ -46,6 +46,41 @@ def _failed_result(*, timeout: bool = False) -> LifecycleResult: ) +def _copy_protected(source: Path, destination: Path) -> str: + """Copy a regular input without following its final component and hash the copy.""" + descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise OSError("protected input is not regular") + digest = hashlib.sha256() + with os.fdopen(descriptor, "rb", closefd=False) as reader, destination.open("wb") as writer: + while chunk := reader.read(1024 * 1024): + digest.update(chunk) + writer.write(chunk) + writer.flush() + os.fsync(writer.fileno()) + destination.chmod(0o400) + if hashlib.sha256(destination.read_bytes()).hexdigest() != digest.hexdigest(): + raise OSError("protected copy changed") + return digest.hexdigest() + finally: + os.close(descriptor) + + +def _installed_file_closure(candidate_python: Path) -> tuple[tuple[str, str, str, str], ...]: + """Measure installed regular files from checker-owned filesystem traversal.""" + environment = candidate_python.parents[1] + rows = [] + for path in sorted(environment.rglob("*")): + if path.is_symlink() or not path.is_file(): + continue + relative = path.relative_to(environment).as_posix() + rows.append(("candidate-environment", "1", relative, + hashlib.sha256(path.read_bytes()).hexdigest())) + return tuple(rows) + + def _required_paths_available( path_inputs: tuple[tuple[Path | None, str], ...], ) -> bool: @@ -287,11 +322,13 @@ def run_lifecycle_matrix( protected_attestation = protected / "candidate-attestation.json" protected_lock = protected / "candidate-runtime.lock" try: - shutil.copy2(candidate_wheel, protected_wheel) - shutil.copy2(candidate_attestation, protected_attestation) - shutil.copy2(candidate_runtime_lock, protected_lock) - for path in (protected_wheel, protected_attestation, protected_lock): - path.chmod(0o400) + _copy_protected(Path(candidate_wheel), protected_wheel) + _copy_protected(Path(candidate_attestation), protected_attestation) + protected_lock_digest = _copy_protected( + Path(candidate_runtime_lock), protected_lock + ) + if protected_lock_digest != candidate_artifact_policy.dependency_lock_sha256: + return _failed_result() candidate_artifact = load_candidate_artifact_provenance( protected_attestation, protected_wheel, candidate_artifact_policy ) @@ -378,4 +415,8 @@ def run_lifecycle_matrix( hashlib.sha256(protected_wheel.read_bytes()).hexdigest(), dependency_digest, candidate_artifact, + protected_lock_digest, + measured_python, + _installed_file_closure(candidate_python), + "pdd-released-checker-v1", ) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index cd0212d70..b2f723eb7 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -6,7 +6,10 @@ import hashlib import ast import json -import re +import os +import shlex +import shutil +import signal import subprocess import sys import tempfile @@ -39,7 +42,9 @@ PurePosixPath("tox.ini"), PurePosixPath("setup.cfg"), ) -PYTEST_PROTECTED_FLAGS = ("--strict-config", "--strict-markers", "-ra") +PYTEST_PROTECTED_FLAGS = ( + "--strict-config", "--strict-markers", "-ra", "-p", "no:cacheprovider" +) _CHECKER_PYTEST_PROBE = Path(__file__).with_name("pytest_probe.py").resolve() @@ -119,7 +124,7 @@ def _local_module_paths( *, code_under_test_paths: frozenset[PurePosixPath] = frozenset(), ) -> tuple[set[PurePosixPath], bool]: - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals,too-many-branches """Resolve repository-local Python imports without executing candidate code.""" try: tree = ast.parse(source) @@ -161,6 +166,20 @@ def _local_module_paths( and node.func.attr == "import_module" ): dynamic_pytest_plugins = True + elif isinstance(node, ast.Call) and ( + isinstance(node.func, ast.Name) + and node.func.id in {"getattr", "exec", "eval", "compile", "run_path", "run_module"} + or isinstance(node.func, ast.Attribute) + and node.func.attr in { + "spec_from_file_location", "load_module", "exec_module", + "run_path", "run_module", + } + ): + dynamic_pytest_plugins = True + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + value = node.value + if isinstance(value, ast.Name) and value.id == "__import__": + dynamic_pytest_plugins = True resolved: set[PurePosixPath] = set() for module in modules: level = len(module) - len(module.lstrip(".")) @@ -373,7 +392,6 @@ def _support_digest( def _config_loads_plugin(root: Path, ref: str) -> bool: """Reject repository-configured plugins until profiles bind plugin identities.""" - pattern = re.compile(r"(?:^|[\s\"'])-p(?:[=\s]+)[A-Za-z0-9_.-]+") for path in PYTEST_CONFIG_PATHS: content = read_git_blob(root, ref, path) if content is None: @@ -382,11 +400,67 @@ def _config_loads_plugin(root: Path, ref: str) -> bool: text = content.decode("utf-8") except UnicodeDecodeError: return True - if pattern.search(text): + try: + tokens = shlex.split(text.replace("=", " = "), comments=True) + except ValueError: return True + for index, token in enumerate(tokens): + if token == "-p" and index + 1 < len(tokens): + return True + if token.startswith("-p=") or (token.startswith("-p") and len(token) > 2): + return True return False +def _managed_subprocess( + command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], + writable_roots: tuple[Path, ...], +) -> tuple[subprocess.CompletedProcess[str], bool]: + # pylint: disable=consider-using-with + """Run an untrusted command in a networkless sandbox and reap its group.""" + argv = command + profile_path = None + if sys.platform == "darwin" and shutil.which("sandbox-exec"): + rules = ["(version 1)", "(allow default)", "(deny network*)", + "(deny file-write*)", + '(allow file-write* (literal "/dev/null"))'] + for item in writable_roots: + rules.append(f'(allow file-write* (subpath "{item.resolve()}"))') + descriptor, profile_name = tempfile.mkstemp(prefix="pdd-sandbox-", suffix=".sb") + os.close(descriptor) + profile_path = Path(profile_name) + profile_path.write_text("\n".join(rules), encoding="utf-8") + argv = ["sandbox-exec", "-f", str(profile_path), *command] + process = subprocess.Popen( + argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=env | { + "PYTHONDONTWRITEBYTECODE": "1", + "TMPDIR": str(writable_roots[0].resolve()), + "TEMP": str(writable_roots[0].resolve()), + "TMP": str(writable_roots[0].resolve()), + }, start_new_session=True, + ) + timed_out = False + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + surviving = False + if not timed_out and os.name != "nt": + try: + os.killpg(process.pid, 0) + surviving = True + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + if profile_path is not None: + profile_path.unlink(missing_ok=True) + return subprocess.CompletedProcess(command, 124 if timed_out else process.returncode, + stdout, stderr), surviving + + def runner_identity_digest( profile: VerificationProfile, *, root: Path | None = None, ref: str = "HEAD" ) -> str: @@ -602,23 +676,23 @@ def _run_test_node( home = temporary / "home" home.mkdir(mode=0o700) junit = temporary / "junit.xml" - try: - result = subprocess.run( - [*command, f"--junitxml={junit}"], - cwd=root, - capture_output=True, - text=True, - check=False, - timeout=timeout_seconds, - env=_pytest_environment(home), - ) - except subprocess.TimeoutExpired: + result, surviving = _managed_subprocess( + [*command, f"--junitxml={junit}"], cwd=root, + timeout=timeout_seconds, env=_pytest_environment(home), + writable_roots=(temporary,), + ) + if result.returncode == 124: return RunnerExecution( node_id, EvidenceOutcome.TIMEOUT, command_digest, "test execution timed out", ) + if surviving: + return RunnerExecution( + node_id, EvidenceOutcome.ERROR, command_digest, + "validator left a surviving process-group descendant", + ) outcome, detail = _junit_outcome( junit, result.returncode, @@ -645,6 +719,8 @@ def _collect_node_ids( "-q", "--strict-config", "--strict-markers", + "-p", + "no:cacheprovider", path.as_posix(), ] runner = _trusted_collection_runner(temporary, root, pytest_args) @@ -661,20 +737,13 @@ def _collect_node_ids( sort_keys=True, ).encode() ).hexdigest() - try: - result = subprocess.run( - command, - cwd=temporary, - capture_output=True, - text=True, - check=False, - timeout=timeout_seconds, - env=_pytest_environment(home) - | { - "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), - }, - ) - except subprocess.TimeoutExpired: + result, surviving = _managed_subprocess( + command, cwd=temporary, timeout=timeout_seconds, + env=_pytest_environment(home) | { + "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), + }, writable_roots=(temporary,), + ) + if result.returncode == 124: return ( RunnerExecution( path.as_posix(), @@ -684,6 +753,13 @@ def _collect_node_ids( ), (), ) + if surviving: + return ( + RunnerExecution( + path.as_posix(), EvidenceOutcome.COLLECTION_ERROR, digest, + "collection left a surviving process-group descendant", + ), (), + ) try: payload = json.loads(collection_output.read_text(encoding="utf-8")) if not isinstance(payload, list) or not all( @@ -697,7 +773,8 @@ def _collect_node_ids( path.as_posix(), EvidenceOutcome.COLLECTION_ERROR, digest, - "trusted collection probe produced no valid node IDs", + "trusted collection probe produced no valid node IDs: " + + (result.stderr or result.stdout)[-500:], ), (), ) @@ -706,7 +783,7 @@ def _collect_node_ids( detail = "zero protected node IDs collected" elif result.returncode != 0: outcome = EvidenceOutcome.COLLECTION_ERROR - detail = "pytest collection failed" + detail = "protected sandbox rejected pytest collection" else: outcome = EvidenceOutcome.PASS detail = f"{len(node_ids)} protected node IDs collected" diff --git a/pdd/sync_core/trust.py b/pdd/sync_core/trust.py index ab1e51248..040f587d5 100644 --- a/pdd/sync_core/trust.py +++ b/pdd/sync_core/trust.py @@ -5,7 +5,6 @@ import base64 import json import os -import stat import subprocess import tempfile from dataclasses import dataclass, replace @@ -19,8 +18,7 @@ Ed25519PrivateKey, Ed25519PublicKey, ) -from filelock import FileLock - +from .descriptor_store import DescriptorStoreError, update_json from .durability import fsync_directory from .types import EvidenceOutcome, ObligationEvidence, UnitId @@ -78,8 +76,7 @@ class FileReplayStore(ReplayStore): """Locked durable nonce ledger located outside candidate-controlled state.""" def __init__(self, path: Path) -> None: - self.path = Path(path).resolve() - self.lock_path = self.path.with_suffix(self.path.suffix + ".lock") + self.path = Path(path).absolute() def _ensure_parent(self) -> None: parent = self.path.parent @@ -124,18 +121,23 @@ def _write(self, payload: dict[str, str]) -> None: def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: """Atomically reject and record every repeated issuer/nonce pair.""" - self._ensure_parent() - with FileLock(str(self.lock_path)): - records = self._load() + def consume_record(records): + if not isinstance(records, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in records.items() + ): + raise AttestationError("replay ledger has invalid records") key = base64.b64encode(f"{issuer}\0{nonce}".encode()).decode("ascii") previous = records.get(key) if previous is not None and previous != attestation_id: raise AttestationError("attestation nonce was replayed") if previous is None: records[key] = attestation_id - self._write(records) - if stat.S_IMODE(self.path.stat().st_mode) != 0o600: - raise AttestationError("replay ledger permissions are unsafe") + return records + try: + update_json(self.path, {}, consume_record) + except DescriptorStoreError as exc: + raise AttestationError(str(exc)) from exc def is_durable(self) -> bool: """Return true for the fsynced external ledger.""" From 520ffcbbe4bb325a6c3d31e6d7246da34cc29e51 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 09:46:50 -0700 Subject: [PATCH 047/128] test(sync): expose remaining verification boundary gaps --- tests/test_sync_core_lifecycle_scenarios.py | 25 +++++++++++++++++++++ tests/test_sync_core_runner.py | 20 +++++++++++++++++ tests/test_sync_core_trust.py | 11 +++++++++ 3 files changed, 56 insertions(+) diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index 05e9b9616..05c0d3462 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -134,6 +134,31 @@ def test_lifecycle_result_binds_recomputable_runtime_measurement() -> None: assert result.measurement_authority == "pdd-released-checker-v1" +def test_lifecycle_measurement_rejects_synthesized_compatibility_fields() -> None: + from pdd.sync_core.certificate import _lifecycle_measurement_complete + + assert _lifecycle_measurement_complete( + LifecycleResult( + 0, 0, 0, 0, 0, 0, + candidate_wheel_sha256="a" * 64, + dependency_environment_digest="b" * 64, + candidate_artifact=object(), + ) + ) is False + + +def test_lifecycle_commands_do_not_use_unsupervised_subprocess_run(monkeypatch) -> None: + from pdd.sync_core import lifecycle + + def forbidden(*_args, **_kwargs): + pytest.fail("lifecycle command bypassed the shared sandbox supervisor") + + monkeypatch.setattr(lifecycle.subprocess, "run", forbidden) + assert lifecycle._candidate_interpreter_identity( + Path(sys.executable), {"PATH": os.environ.get("PATH", "")} + ) is not None + + def test_lifecycle_matrix_fails_closed_without_hash_pinned_wheelhouse( tmp_path, ) -> None: diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index c232a1e5b..da75b852e 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -2,6 +2,7 @@ import os import subprocess +import sys from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -639,3 +640,22 @@ def test_surviving_validator_descendant_cannot_pass(tmp_path) -> None: os.kill(int(pid_file.read_text()), 9) except ProcessLookupError: pass + + +def test_managed_subprocess_fails_closed_without_supported_os_sandbox( + tmp_path, monkeypatch +) -> None: + from pdd.sync_core.runner import _managed_subprocess + + monkeypatch.setattr("pdd.sync_core.runner.sys.platform", "freebsd14") + result, surviving = _managed_subprocess( + [sys.executable, "-c", "print('must not execute')"], + cwd=tmp_path, + timeout=5, + env={}, + writable_roots=(tmp_path,), + ) + assert result.returncode != 0 + assert "unsupported sandbox platform" in result.stderr + assert "must not execute" not in result.stdout + assert surviving is False diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index b4dc382fa..817a9298d 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -250,3 +250,14 @@ def test_file_replay_store_rejects_symlink_lock(tmp_path) -> None: store = FileReplayStore(path) with pytest.raises(AttestationError, match="unsafe"): store.consume("trusted-ci", "nonce", "attestation") + + +def test_file_replay_store_rejects_symlinked_ancestor(tmp_path) -> None: + protected = tmp_path / "protected" + protected.mkdir(mode=0o700) + outside = tmp_path / "outside" + outside.mkdir(mode=0o700) + (protected / "linked").symlink_to(outside, target_is_directory=True) + store = FileReplayStore(protected / "linked" / "nested" / "replay.json") + with pytest.raises(AttestationError, match="unsafe"): + store.consume("trusted-ci", "nonce", "attestation") From 73f968bfde11727fba14bb5b3f6ebcba33436fc7 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 09:57:55 -0700 Subject: [PATCH 048/128] fix(sync): require portable verifiable boundaries --- pdd/sync_core/artifact_provenance.py | 15 ++- pdd/sync_core/certificate.py | 58 +++++----- pdd/sync_core/descriptor_store.py | 35 ++++-- pdd/sync_core/lifecycle.py | 116 ++++++-------------- pdd/sync_core/runner.py | 48 +------- pdd/sync_core/supervisor.py | 74 +++++++++++++ pdd/sync_core/trust.py | 13 ++- tests/test_sync_core_certificate.py | 12 ++ tests/test_sync_core_lifecycle_scenarios.py | 10 +- 9 files changed, 207 insertions(+), 174 deletions(-) create mode 100644 pdd/sync_core/supervisor.py diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py index cbad5cc94..59fb31902 100644 --- a/pdd/sync_core/artifact_provenance.py +++ b/pdd/sync_core/artifact_provenance.py @@ -146,12 +146,17 @@ def consume_record(records): ) records.append(list(key)) return records - try: - update_json(path, [], consume_record) - except DescriptorStoreError as exc: + error = None + for _attempt in range(3): + try: + update_json(path, [], consume_record) + return + except DescriptorStoreError as exc: + error = exc + if error is not None: raise CandidateArtifactProvenanceError( - f"candidate replay ledger is unsafe: {exc}" - ) from exc + f"candidate replay ledger is unsafe: {error}" + ) from error def _read_durable_records(self, path: Path) -> list[list[str]]: if path.exists(): diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index 317b3b095..949065d6d 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -560,7 +560,8 @@ def _aggregate_counts(reports: list[dict[str, Any]]) -> dict[str, int]: def _scan_predicate( - counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int] + counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int], + *, require_measurement: bool = True, ) -> bool: managed = counts["managed_units"] baseline_failures = ("DRIFTED", "UNBASELINED", "CORRUPT") @@ -594,14 +595,35 @@ def _scan_predicate( for character in lifecycle.dependency_environment_digest ) and lifecycle.candidate_artifact is not None + and (not require_measurement or _lifecycle_measurement_complete(lifecycle)) and extra["nightly_observation_complete"] == 1 ) +def _lifecycle_measurement_complete(lifecycle: LifecycleResult) -> bool: + """Require checker-owned measured closure fields; never synthesize them.""" + hexadecimal = set("0123456789abcdef") + interpreter_keys = {"implementation", "version", "abi", "platform"} + return ( + len(lifecycle.runtime_lock_sha256) == 64 + and set(lifecycle.runtime_lock_sha256) <= hexadecimal + and isinstance(lifecycle.interpreter, dict) + and set(lifecycle.interpreter) == interpreter_keys + and all(isinstance(value, str) and value for value in lifecycle.interpreter.values()) + and bool(lifecycle.installed_files) + and all(len(item) == 4 and all(isinstance(value, str) and value for value in item) + and len(item[3]) == 64 and set(item[3]) <= hexadecimal + for item in lifecycle.installed_files) + and lifecycle.measurement_authority == "pdd-released-checker-v1" + ) + + def _predicate( - counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int] + counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int], + *, require_measurement: bool = True, ) -> bool: - return _scan_predicate(counts, lifecycle, extra) and ( + return _scan_predicate(counts, lifecycle, extra, + require_measurement=require_measurement) and ( extra["nightly_streak"] >= extra["required_nightly_streak"] ) @@ -823,8 +845,12 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] != expected_evidence_coverage ): return False, False - scan_ok = all(report_results) and _scan_predicate(aggregate, lifecycle, extra) - ok = all(report_results) and _predicate(aggregate, lifecycle, extra) + scan_ok = all(report_results) and _scan_predicate( + aggregate, lifecycle, extra, require_measurement=schema_version == 4 + ) + ok = all(report_results) and _predicate( + aggregate, lifecycle, extra, require_measurement=schema_version == 4 + ) return scan_ok, ok @@ -894,28 +920,6 @@ def _build_global_certificate_from_targets( if candidate_artifact_valid else replace(options.lifecycle_result, candidate_artifact=None) ) - if candidate_artifact_valid and lifecycle.candidate_artifact is not None: - policy = options.candidate_artifact_policy - lifecycle = replace( - lifecycle, - runtime_lock_sha256=( - lifecycle.runtime_lock_sha256 or policy.dependency_lock_sha256 - ), - interpreter=lifecycle.interpreter or { - "implementation": policy.python_implementation, - "version": policy.python_version, - "abi": policy.python_abi, - "platform": policy.python_platform, - }, - installed_files=lifecycle.installed_files or (( - "candidate-wheel", "1", "candidate.whl", - lifecycle.candidate_wheel_sha256, - ),), - measurement_authority=( - lifecycle.measurement_authority - or options.checker_identity.workflow_identity - ), - ) body: dict[str, Any] = { "schema_version": 4, "checked_at": datetime.now(timezone.utc).isoformat(), diff --git a/pdd/sync_core/descriptor_store.py b/pdd/sync_core/descriptor_store.py index 604007616..7c81dc3e2 100644 --- a/pdd/sync_core/descriptor_store.py +++ b/pdd/sync_core/descriptor_store.py @@ -34,22 +34,37 @@ def _unlock(descriptor: int) -> None: def _safe_parent(path: Path) -> tuple[int, os.stat_result]: - """Open the ledger parent once and return its stable directory identity.""" + """Traverse to the ledger parent from an anchored root without following links.""" if path.name in {"", ".", ".."} or path.is_absolute() is False: path = path.absolute() - parent = path.parent - parent.mkdir(mode=0o700, parents=True, exist_ok=True) flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) try: - descriptor = os.open(parent, flags) - except OSError as exc: + descriptor = os.open(path.anchor, flags) + for component in path.parent.parts[1:]: + if component in {"", ".", ".."}: + raise DescriptorStoreError("replay ledger parent is unsafe") + try: + child = os.open(component, flags, dir_fd=descriptor) + except FileNotFoundError: + os.mkdir(component, mode=0o700, dir_fd=descriptor) + child = os.open(component, flags, dir_fd=descriptor) + metadata = os.fstat(child) + if not stat.S_ISDIR(metadata.st_mode) or ( + hasattr(os, "getuid") and metadata.st_uid not in {0, os.getuid()} + ): + os.close(child) + raise DescriptorStoreError("replay ledger parent is unsafe") + os.close(descriptor) + descriptor = child + except (OSError, NotImplementedError) as exc: + try: + os.close(descriptor) + except (UnboundLocalError, OSError): + pass raise DescriptorStoreError("replay ledger parent is unsafe") from exc opened = os.fstat(descriptor) - lexical = os.lstat(parent) if ( not stat.S_ISDIR(opened.st_mode) - or stat.S_ISLNK(lexical.st_mode) - or (opened.st_dev, opened.st_ino) != (lexical.st_dev, lexical.st_ino) or (hasattr(os, "getuid") and opened.st_uid != os.getuid()) or stat.S_IMODE(opened.st_mode) & 0o077 ): @@ -78,7 +93,7 @@ def _read_json(parent_fd: int, name: str, empty: Any) -> Any: os.stat(name, dir_fd=parent_fd, follow_symlinks=False) except FileNotFoundError: return empty - raise + descriptor = _open_relative(parent_fd, name, os.O_RDONLY) try: metadata = os.fstat(descriptor) if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600: @@ -130,7 +145,7 @@ def update_json( replacement = update(payload) if replacement is not None: _write_json(parent_fd, path.name, replacement) - current = os.stat(path.parent, follow_symlinks=False) + current = os.fstat(parent_fd) if (current.st_dev, current.st_ino) != (identity.st_dev, identity.st_ino): raise DescriptorStoreError("replay ledger parent changed") return payload if replacement is None else replacement diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index fefe5548f..41588199b 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -20,6 +20,7 @@ from .certificate import LifecycleResult from .isolation import untrusted_child_environment from .scenario_contract import REQUIRED_SCENARIOS +from .supervisor import run_supervised def _isolated_lifecycle_environment(home: Path) -> dict[str, str]: @@ -126,66 +127,36 @@ def _normalized_results(payload: Any) -> dict[str, dict[str, Any]]: return results -def _dependency_environment_digest(candidate_python: Path, isolated: dict[str, str]) -> str: - """Hash the installed distribution set visible to the candidate wheel.""" - probe = subprocess.run( - [ - str(candidate_python), - "-c", - ( - "import importlib.metadata as m, json;" - "rows=[];" - "\nfor d in m.distributions():" - "\n rows.append({" - "'name': d.metadata.get('Name', '')," - "'version': d.version," - "'files': sorted(" - "(str(f), getattr(getattr(f, 'hash', None), 'value', ''), f.size or 0)" - " for f in (d.files or ()))" - "})" - "\nprint(json.dumps(sorted(rows, key=lambda r: (r['name'].lower(), r['version']))," - " sort_keys=True, separators=(',', ':')))" - ), - ], - capture_output=True, - text=True, - check=False, - env=isolated, - ) - if probe.returncode != 0: - return "" - return hashlib.sha256(probe.stdout.encode()).hexdigest() +def _dependency_environment_digest(candidate_python: Path, _isolated: dict[str, str]) -> str: + """Hash the checker-owned installed-file closure without candidate startup.""" + closure = _installed_file_closure(candidate_python) + return hashlib.sha256(json.dumps(closure, separators=(",", ":")).encode()).hexdigest() def _candidate_interpreter_identity( candidate_python: Path, isolated: dict[str, str] ) -> dict[str, str] | None: - """Measure the interpreter and first compatible wheel tag actually executed.""" - probe = subprocess.run( - [ - str(candidate_python), - "-I", - "-c", - ( - "import json, platform; from packaging.tags import sys_tags;" - "tag=next(sys_tags()); print(json.dumps({" - "'implementation':platform.python_implementation()," - "'version':platform.python_version(),'abi':tag.abi," - "'platform':tag.platform},sort_keys=True))" - ), - ], - capture_output=True, - text=True, - check=False, - env=isolated, + """Measure the checker interpreter used to create the venv, before site startup.""" + del candidate_python, isolated + import platform # pylint: disable=import-outside-toplevel + from packaging.tags import sys_tags # pylint: disable=import-outside-toplevel + tag = next(sys_tags()) + return {"implementation": platform.python_implementation(), + "version": platform.python_version(), "abi": tag.abi, + "platform": tag.platform} + + +def _lifecycle_command(command: list[str], temporary: Path, home: Path, + timeout: int = 1200) -> subprocess.CompletedProcess[str]: + """Route every lifecycle command through the shared fail-closed supervisor.""" + result, surviving = run_supervised( + command, cwd=temporary, timeout=timeout, + env=_isolated_lifecycle_environment(home), writable_roots=(temporary,) ) - if probe.returncode != 0: - return None - try: - measured = json.loads(probe.stdout) - except json.JSONDecodeError: - return None - return measured if isinstance(measured, dict) else None + if surviving: + return subprocess.CompletedProcess(command, 125, result.stdout, + result.stderr + "\nsurviving descendant") + return result def _combined_candidate_lock(temporary: Path, runtime_lock: Path, wheel: Path) -> Path | None: @@ -223,19 +194,15 @@ def _install_candidate_wheel( return None environment = temporary / "candidate-venv" isolated = _isolated_lifecycle_environment(home) - created = subprocess.run( - [sys.executable, "-m", "venv", str(environment)], - capture_output=True, - text=True, - check=False, - env=isolated, + created = _lifecycle_command( + [sys.executable, "-m", "venv", str(environment)], temporary, home ) candidate_python = environment / ( "Scripts/python.exe" if os.name == "nt" else "bin/python" ) if created.returncode != 0: return None - installed = subprocess.run( + installed = _lifecycle_command( [ str(candidate_python), "-m", @@ -251,19 +218,12 @@ def _install_candidate_wheel( "-r", str(combined_lock), ], - capture_output=True, - text=True, - check=False, - env=isolated, + temporary, home, ) if installed.returncode != 0: return None - proved = subprocess.run( - [str(candidate_python), "-I", "-m", "pdd.cli", "--help"], - capture_output=True, - text=True, - check=False, - env=isolated, + proved = _lifecycle_command( + [str(candidate_python), "-I", "-m", "pdd.cli", "--help"], temporary, home ) if proved.returncode != 0: return None @@ -373,16 +333,10 @@ def run_lifecycle_matrix( "--candidate-python", str(candidate_python), ] - try: - completed = subprocess.run( - command, - capture_output=True, - text=True, - check=False, - timeout=timeout_seconds, - env=_isolated_lifecycle_environment(temporary / "home"), - ) - except subprocess.TimeoutExpired: + completed = _lifecycle_command( + command, temporary, temporary / "home", timeout_seconds + ) + if completed.returncode == 124: return _failed_result(timeout=True) try: results = _normalized_results(json.loads(output.read_text(encoding="utf-8"))) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index b2f723eb7..7c0767bb6 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -6,10 +6,7 @@ import hashlib import ast import json -import os import shlex -import shutil -import signal import subprocess import sys import tempfile @@ -33,6 +30,7 @@ VerificationObligation, VerificationProfile, ) +from .supervisor import run_supervised TRUSTED_RUNNER_VERSION = "pdd-trusted-runner-v2" @@ -416,49 +414,9 @@ def _managed_subprocess( command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], writable_roots: tuple[Path, ...], ) -> tuple[subprocess.CompletedProcess[str], bool]: - # pylint: disable=consider-using-with """Run an untrusted command in a networkless sandbox and reap its group.""" - argv = command - profile_path = None - if sys.platform == "darwin" and shutil.which("sandbox-exec"): - rules = ["(version 1)", "(allow default)", "(deny network*)", - "(deny file-write*)", - '(allow file-write* (literal "/dev/null"))'] - for item in writable_roots: - rules.append(f'(allow file-write* (subpath "{item.resolve()}"))') - descriptor, profile_name = tempfile.mkstemp(prefix="pdd-sandbox-", suffix=".sb") - os.close(descriptor) - profile_path = Path(profile_name) - profile_path.write_text("\n".join(rules), encoding="utf-8") - argv = ["sandbox-exec", "-f", str(profile_path), *command] - process = subprocess.Popen( - argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - env=env | { - "PYTHONDONTWRITEBYTECODE": "1", - "TMPDIR": str(writable_roots[0].resolve()), - "TEMP": str(writable_roots[0].resolve()), - "TMP": str(writable_roots[0].resolve()), - }, start_new_session=True, - ) - timed_out = False - try: - stdout, stderr = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - timed_out = True - os.killpg(process.pid, signal.SIGKILL) - stdout, stderr = process.communicate() - surviving = False - if not timed_out and os.name != "nt": - try: - os.killpg(process.pid, 0) - surviving = True - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - if profile_path is not None: - profile_path.unlink(missing_ok=True) - return subprocess.CompletedProcess(command, 124 if timed_out else process.returncode, - stdout, stderr), surviving + return run_supervised(command, cwd=cwd, timeout=timeout, env=env, + writable_roots=writable_roots) def runner_identity_digest( diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py new file mode 100644 index 000000000..16c03c14d --- /dev/null +++ b/pdd/sync_core/supervisor.py @@ -0,0 +1,74 @@ +"""Fail-closed OS sandbox and complete process-group supervision.""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import sys +import tempfile +from pathlib import Path + + +def _sandbox_command( + command: list[str], writable_roots: tuple[Path, ...] +) -> tuple[list[str], Path | None]: + """Return an explicitly detected macOS/Linux sandbox command.""" + if sys.platform == "darwin" and shutil.which("sandbox-exec"): + rules = ["(version 1)", "(allow default)", "(deny network*)", "(deny file-write*)", + '(allow file-write* (literal "/dev/null"))'] + for item in writable_roots: + rules.append(f'(allow file-write* (subpath "{item.resolve()}"))') + descriptor, name = tempfile.mkstemp(prefix="pdd-sandbox-", suffix=".sb") + os.close(descriptor) + profile = Path(name) + profile.write_text("\n".join(rules), encoding="utf-8") + return ["sandbox-exec", "-f", str(profile), *command], profile + if sys.platform.startswith("linux") and shutil.which("bwrap"): + argv = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session", + "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc"] + for item in writable_roots: + resolved = str(item.resolve()) + argv.extend(("--bind", resolved, resolved)) + return [*argv, "--", *command], None + raise RuntimeError("unsupported sandbox platform or mechanism") + + +def run_supervised( + command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], + writable_roots: tuple[Path, ...], +) -> tuple[subprocess.CompletedProcess[str], bool]: + """Run inside a supported networkless sandbox and kill the whole process group.""" + # pylint: disable=consider-using-with + try: + argv, profile = _sandbox_command(command, writable_roots) + except RuntimeError as exc: + return subprocess.CompletedProcess(command, 125, "", str(exc)), False + process = subprocess.Popen( + argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=env | {"PYTHONDONTWRITEBYTECODE": "1", + "TMPDIR": str(writable_roots[0].resolve()), + "TEMP": str(writable_roots[0].resolve()), + "TMP": str(writable_roots[0].resolve())}, + start_new_session=True, + ) + timed_out = False + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + surviving = False + if not timed_out and os.name != "nt": + try: + os.killpg(process.pid, 0) + surviving = True + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + if profile is not None: + profile.unlink(missing_ok=True) + return subprocess.CompletedProcess(command, 124 if timed_out else process.returncode, + stdout, stderr), surviving diff --git a/pdd/sync_core/trust.py b/pdd/sync_core/trust.py index 040f587d5..d3ba203ea 100644 --- a/pdd/sync_core/trust.py +++ b/pdd/sync_core/trust.py @@ -134,10 +134,15 @@ def consume_record(records): if previous is None: records[key] = attestation_id return records - try: - update_json(self.path, {}, consume_record) - except DescriptorStoreError as exc: - raise AttestationError(str(exc)) from exc + error = None + for _attempt in range(3): + try: + update_json(self.path, {}, consume_record) + return + except DescriptorStoreError as exc: + error = exc + if error is not None: + raise AttestationError(str(error)) from error def is_durable(self) -> bool: """Return true for the fsynced external ledger.""" diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 6c9f6e0ff..3c5eb6842 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -48,6 +48,13 @@ "cp312", "manylinux_2_17_x86_64", ) +MEASURED_CLOSURE = { + "runtime_lock_sha256": DEPENDENCY_ENVIRONMENT_DIGEST, + "interpreter": {"implementation": "CPython", "version": "3.12.3", + "abi": "cp312", "platform": "manylinux_2_17_x86_64"}, + "installed_files": (("pdd", "1.0", "pdd/__init__.py", "f" * 64),), + "measurement_authority": "pdd-released-checker-v1", +} @pytest.fixture(autouse=True) @@ -303,6 +310,7 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, candidate_artifact=_candidate_artifact(), + **MEASURED_CLOSURE, ) certificate = build_global_certificate( ( @@ -483,6 +491,7 @@ def test_certificate_acceptance_binds_freshness_issuer_and_exact_refs( candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, candidate_artifact=_candidate_artifact(), + **MEASURED_CLOSURE, ), nightly, checker_identity=CHECKER, @@ -644,6 +653,7 @@ def test_unsigned_nightly_rows_cannot_fabricate_temporal_proof( candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, candidate_artifact=_candidate_artifact(), + **MEASURED_CLOSURE, ), nightly, checker_identity=CHECKER, @@ -692,6 +702,7 @@ def test_signed_nightly_rows_without_observations_do_not_extend_streak( candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, dependency_environment_digest=DEPENDENCY_ENVIRONMENT_DIGEST, candidate_artifact=_candidate_artifact(), + **MEASURED_CLOSURE, ), nightly, checker_identity=CHECKER, @@ -743,6 +754,7 @@ def test_signed_nightly_rows_with_untrusted_candidate_artifacts_do_not_extend_st 0, candidate_wheel_sha256=CANDIDATE_WHEEL_SHA256, candidate_artifact=_candidate_artifact(), + **MEASURED_CLOSURE, ), nightly, checker_identity=CHECKER, diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index 05c0d3462..85648269a 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -233,7 +233,10 @@ def fake_run(command, **kwargs): ) return subprocess.CompletedProcess(command, 0, "ok", "") - monkeypatch.setattr("pdd.sync_core.lifecycle.subprocess.run", fake_run) + monkeypatch.setattr( + "pdd.sync_core.lifecycle._lifecycle_command", + lambda command, *_args, **_kwargs: fake_run(command), + ) installed = _install_candidate_wheel( tmp_path, tmp_path / "home", @@ -273,7 +276,10 @@ def fake_run(command, **_kwargs): ) return subprocess.CompletedProcess(command, 0, "ok", "") - monkeypatch.setattr("pdd.sync_core.lifecycle.subprocess.run", fake_run) + monkeypatch.setattr( + "pdd.sync_core.lifecycle._lifecycle_command", + lambda command, *_args, **_kwargs: fake_run(command), + ) assert _install_candidate_wheel( tmp_path, tmp_path / "home", From 07015247c0382419e42ed1c3207e9bcf2892edd1 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 10:30:57 -0700 Subject: [PATCH 049/128] test(sync): define protected replay root boundary --- tests/test_sync_core_descriptor_store.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/test_sync_core_descriptor_store.py diff --git a/tests/test_sync_core_descriptor_store.py b/tests/test_sync_core_descriptor_store.py new file mode 100644 index 000000000..059115f2b --- /dev/null +++ b/tests/test_sync_core_descriptor_store.py @@ -0,0 +1,49 @@ +"""Regression tests for descriptor-anchored replay ledger storage.""" + +from __future__ import annotations + +import json + +import pytest + +from pdd.sync_core.descriptor_store import DescriptorStoreError, update_json + + +def test_checker_owned_replay_root_ignores_platform_temp_ancestor_mode( + tmp_path, +) -> None: + """Only the configured checker-owned root begins the protected boundary.""" + platform_temp = tmp_path / "platform-temp" + platform_temp.mkdir(mode=0o777) + platform_temp.chmod(0o777) + replay_root = platform_temp / "checker-owned" + replay_root.mkdir(mode=0o700) + ledger = replay_root / "nested" / "replay.json" + + update_json( + ledger, + [], + lambda records: [*records, "accepted"], + trust_root=replay_root, + ) + + assert json.loads(ledger.read_text(encoding="utf-8")) == ["accepted"] + + +def test_checker_owned_replay_root_rejects_symlinked_descendant(tmp_path) -> None: + """Descriptor traversal below the trust root must never follow symlinks.""" + replay_root = tmp_path / "checker-owned" + replay_root.mkdir(mode=0o700) + outside = tmp_path / "outside" + outside.mkdir() + (replay_root / "linked").symlink_to(outside, target_is_directory=True) + + with pytest.raises(DescriptorStoreError, match="parent is unsafe"): + update_json( + replay_root / "linked" / "replay.json", + [], + lambda records: records, + trust_root=replay_root, + ) + + assert not (outside / "replay.json").exists() From 3b6a06b5a7ce1c2986937c6ba3d8e40059c48a50 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 10:37:28 -0700 Subject: [PATCH 050/128] fix(sync): anchor candidate replay storage at protected root --- pdd/commands/sync_core.py | 2 +- pdd/sync_core/artifact_provenance.py | 2 +- pdd/sync_core/descriptor_store.py | 95 ++++++++++++++++++++++------ 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py index 942a1484a..ae13e6174 100644 --- a/pdd/commands/sync_core.py +++ b/pdd/commands/sync_core.py @@ -206,7 +206,7 @@ def certify( ) if replay_ledger.exists() and not replay_ledger.is_dir(): raise ValueError("global --replay-ledger must be a directory") - replay_ledger.mkdir(parents=True, exist_ok=True) + replay_ledger.mkdir(mode=0o700, parents=True, exist_ok=True) candidate_policy = candidate_artifact_policy_from_environment() candidate_policy.replay_ledger_path = ( replay_ledger / "candidate-artifacts.json" diff --git a/pdd/sync_core/artifact_provenance.py b/pdd/sync_core/artifact_provenance.py index 59fb31902..9dade0598 100644 --- a/pdd/sync_core/artifact_provenance.py +++ b/pdd/sync_core/artifact_provenance.py @@ -149,7 +149,7 @@ def consume_record(records): error = None for _attempt in range(3): try: - update_json(path, [], consume_record) + update_json(path, [], consume_record, trust_root=path.parent) return except DescriptorStoreError as exc: error = exc diff --git a/pdd/sync_core/descriptor_store.py b/pdd/sync_core/descriptor_store.py index 7c81dc3e2..97b13d2db 100644 --- a/pdd/sync_core/descriptor_store.py +++ b/pdd/sync_core/descriptor_store.py @@ -33,16 +33,22 @@ def _unlock(descriptor: int) -> None: fcntl.flock(descriptor, fcntl.LOCK_UN) -def _safe_parent(path: Path) -> tuple[int, os.stat_result]: - """Traverse to the ledger parent from an anchored root without following links.""" - if path.name in {"", ".", ".."} or path.is_absolute() is False: - path = path.absolute() +def _safe_directory(metadata: os.stat_result) -> bool: + """Return whether metadata describes a private checker-owned directory.""" + return ( + stat.S_ISDIR(metadata.st_mode) + and (not hasattr(os, "getuid") or metadata.st_uid == os.getuid()) + and not stat.S_IMODE(metadata.st_mode) & 0o077 + ) + + +def _legacy_safe_parent(path: Path) -> tuple[int, os.stat_result]: + """Preserve strict filesystem-root validation for unanchored callers.""" flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = -1 try: descriptor = os.open(path.anchor, flags) for component in path.parent.parts[1:]: - if component in {"", ".", ".."}: - raise DescriptorStoreError("replay ledger parent is unsafe") try: child = os.open(component, flags, dir_fd=descriptor) except FileNotFoundError: @@ -56,18 +62,63 @@ def _safe_parent(path: Path) -> tuple[int, os.stat_result]: raise DescriptorStoreError("replay ledger parent is unsafe") os.close(descriptor) descriptor = child - except (OSError, NotImplementedError) as exc: - try: + except (OSError, NotImplementedError, DescriptorStoreError) as exc: + if descriptor >= 0: os.close(descriptor) - except (UnboundLocalError, OSError): - pass + if isinstance(exc, DescriptorStoreError): + raise raise DescriptorStoreError("replay ledger parent is unsafe") from exc opened = os.fstat(descriptor) - if ( - not stat.S_ISDIR(opened.st_mode) - or (hasattr(os, "getuid") and opened.st_uid != os.getuid()) - or stat.S_IMODE(opened.st_mode) & 0o077 - ): + if not _safe_directory(opened): + os.close(descriptor) + raise DescriptorStoreError("replay ledger parent is unsafe") + return descriptor, opened + + +def _safe_parent( + path: Path, trust_root: Path | None +) -> tuple[int, os.stat_result]: + """Traverse below a checker-owned trust root without following links.""" + path = path.absolute() + if path.name in {"", ".", ".."}: + raise DescriptorStoreError("replay ledger parent is unsafe") + if trust_root is None: + return _legacy_safe_parent(path) + trust_root = trust_root.absolute() + try: + relative_parent = path.parent.relative_to(trust_root) + except ValueError as exc: + raise DescriptorStoreError("replay ledger escapes protected root") from exc + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = -1 + try: + trust_root.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor = os.open(trust_root, flags) + root_metadata = os.fstat(descriptor) + if not _safe_directory(root_metadata): + raise DescriptorStoreError("replay ledger root is unsafe") + for component in relative_parent.parts: + if component in {"", ".", ".."}: + raise DescriptorStoreError("replay ledger parent is unsafe") + try: + child = os.open(component, flags, dir_fd=descriptor) + except FileNotFoundError: + os.mkdir(component, mode=0o700, dir_fd=descriptor) + child = os.open(component, flags, dir_fd=descriptor) + metadata = os.fstat(child) + if not _safe_directory(metadata): + os.close(child) + raise DescriptorStoreError("replay ledger parent is unsafe") + os.close(descriptor) + descriptor = child + except (OSError, NotImplementedError, DescriptorStoreError) as exc: + if descriptor >= 0: + os.close(descriptor) + if isinstance(exc, DescriptorStoreError): + raise + raise DescriptorStoreError("replay ledger parent is unsafe") from exc + opened = os.fstat(descriptor) + if not _safe_directory(opened): os.close(descriptor) raise DescriptorStoreError("replay ledger parent is unsafe") return descriptor, opened @@ -128,11 +179,17 @@ def _write_json(parent_fd: int, name: str, payload: Any) -> None: def update_json( - path: Path, empty: Any, update: Callable[[Any], Any] + path: Path, + empty: Any, + update: Callable[[Any], Any], + *, + trust_root: Path | None = None, ) -> Any: - """Lock and update one JSON ledger using only its held parent descriptor.""" - path = Path(path) - parent_fd, identity = _safe_parent(path) + """Update JSON below an explicit checker-owned descriptor trust anchor.""" + path = Path(path).absolute() + parent_fd, identity = _safe_parent( + path, Path(trust_root) if trust_root is not None else None + ) lock_name = f"{path.name}.lock" lock_fd = -1 try: From b9e154b1ca4243d0e7ae6dd868947be0ace2bb72 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 11:27:07 -0700 Subject: [PATCH 051/128] test(sync): expose round 11 trust boundary gaps --- tests/test_sync_core_certificate.py | 21 +++++++++++++- tests/test_sync_core_includes.py | 22 +++++++++++++++ tests/test_sync_core_reporting.py | 14 +++++++++ tests/test_sync_core_runner.py | 32 +++++++++++++++++++++ tests/test_sync_core_supervisor.py | 39 ++++++++++++++++++++++++++ tests/test_sync_determine_operation.py | 14 +++++++++ 6 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 tests/test_sync_core_supervisor.py diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 3c5eb6842..91e93de79 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -1,6 +1,7 @@ """Tests for the signed cross-repository certificate predicate.""" import base64 +import hashlib import json import subprocess from datetime import datetime, timedelta, timezone @@ -104,6 +105,7 @@ def _nightly( include_observation=True, candidate_artifact=None, start=None, + schema_version=4, ): start = start or datetime.now(timezone.utc) - timedelta(days=count - 1) rows = [] @@ -150,9 +152,10 @@ def _nightly( "candidate_wheel_sha256": CANDIDATE_WHEEL_SHA256, "dependency_environment_digest": DEPENDENCY_ENVIRONMENT_DIGEST, "candidate_artifact": row_candidate_artifact.payload(), + **MEASURED_CLOSURE, } body = { - "schema_version": 3, + "schema_version": schema_version, "checked_at": checked_at.isoformat(), "checker": CHECKER.payload(), "candidate_artifact_policy": CANDIDATE_POLICY.identity(), @@ -328,6 +331,10 @@ def test_complete_global_predicate_is_signed_and_verifiable(tmp_path, monkeypatc signer=signer, ) assert certificate["ok"] is True + measured_digest = hashlib.sha256( + json.dumps(MEASURED_CLOSURE["installed_files"], separators=(",", ":")).encode() + ).hexdigest() + assert certificate["lifecycle"]["dependency_environment_digest"] == measured_digest assert certificate["counts"]["managed_units"] == 2 assert certificate["counts"]["verification_profile_coverage"] == 100 assert certificate["counts"]["trusted_current_evidence_coverage"] == 100 @@ -789,6 +796,18 @@ def test_historical_nightly_candidate_artifact_uses_row_checked_at(tmp_path) -> ) == 1 +def test_schema_v3_history_cannot_extend_schema_v4_nightly_streak(tmp_path) -> None: + signer = AttestationSigner("certificate-ci", b"g" * 32) + nightly = tmp_path / "nightly.jsonl" + _nightly(nightly, signer, schema_version=3) + assert _nightly_streak( + nightly, + _NightlyVerificationPolicy( + signer.public_key_bytes(), signer.issuer, (), CHECKER, CANDIDATE_POLICY + ), + ) == 0 + + def test_nightly_lineage_requires_identity_and_ancestry(tmp_path) -> None: targets = [] reports = [] diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py index b18460d87..e843c67ed 100644 --- a/tests/test_sync_core_includes.py +++ b/tests/test_sync_core_includes.py @@ -147,6 +147,28 @@ def test_optional_missing_include_is_bound_but_not_failed(tmp_path) -> None: assert closure.artifacts == () +@pytest.mark.parametrize( + "attribute,value", + [ + ("path", "docs/optional.md"), + ("select", "optional"), + ("query", "please expand this"), + ], +) +def test_boolean_attribute_names_inside_quotes_do_not_make_include_optional( + tmp_path, attribute, value +) -> None: + (tmp_path / "prompts").mkdir() + attrs = f'{attribute}="{value}"' + if attribute != "path": + attrs = f'path="missing.md" {attrs}' + (tmp_path / "prompts/a.prompt").write_text( + f"", encoding="utf-8" + ) + with pytest.raises(IncludeGraphError, match="missing"): + build_include_closure(PurePosixPath("prompts/a.prompt"), PathPolicy(tmp_path)) + + def test_query_include_marks_closure_nondeterministic(tmp_path) -> None: (tmp_path / "prompts").mkdir() (tmp_path / "docs").mkdir() diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index 7ac0fb65a..9cd118ef4 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -336,6 +336,20 @@ def test_repository_identity_does_not_activate_enforcement_without_policy( assert canonical_sync_enabled(root) is False +@pytest.mark.parametrize("protected_ref", ["missing-ref", "HEAD"]) +def test_explicit_protected_mode_invalid_trust_input_fails_closed( + tmp_path, monkeypatch, protected_ref +) -> None: + root, _commit = _repository(tmp_path) + if protected_ref == "HEAD": + (root / ".pdd/sync-policy.json").write_text("{not-json") + _git(root, "add", ".pdd/sync-policy.json") + _git(root, "commit", "-q", "-m", "malformed protected policy") + monkeypatch.setenv("PDD_SYNC_PROTECTED_BASE_SHA", protected_ref) + with pytest.raises(ValueError, match="protected"): + canonical_sync_enabled(root) + + def test_code_drift_cannot_reuse_old_attestation(tmp_path) -> None: root, commit = _repository(tmp_path) _finalize_trusted_baseline(root, commit) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index da75b852e..ca1d99414 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -177,6 +177,38 @@ def test_candidate_modified_product_code_can_be_certified_by_protected_tests(tmp assert executions[0].outcome is EvidenceOutcome.PASS +def test_dirty_declared_product_code_cannot_receive_committed_head_evidence(tmp_path) -> None: + root, _initial = _repository( + tmp_path, "import product\ndef test_widget(): assert product.expected() == 1\n" + ) + (root / "product.py").write_text("def expected(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "product") + head = _git(root, "rev-parse", "HEAD") + (root / "product.py").write_text("def expected(): return 2\n") + _envelope, executions = _run( + root, head, head, (PurePosixPath("product.py"),) + ) + assert executions[0].outcome is EvidenceOutcome.QUARANTINED + assert "dirty" in executions[0].detail + + +def test_declared_product_reflection_is_not_an_unbound_test_loader(tmp_path) -> None: + root, _initial = _repository( + tmp_path, "import product\ndef test_widget(): assert product.expected() == 1\n" + ) + (root / "product.py").write_text( + "def expected():\n return getattr(type('X', (), {'value': 1}), 'value')\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "reflective product") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run( + root, head, head, (PurePosixPath("product.py"),) + ) + assert executions[0].outcome is EvidenceOutcome.PASS + + def test_candidate_modified_helper_outside_tests_is_protected(tmp_path) -> None: root, _initial = _repository( tmp_path, diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py new file mode 100644 index 000000000..36997ab2c --- /dev/null +++ b/tests/test_sync_core_supervisor.py @@ -0,0 +1,39 @@ +"""Adversarial tests for complete protected subprocess supervision.""" + +import os +import shutil +import sys +import time +from pathlib import Path + +import pytest + +from pdd.sync_core.supervisor import run_supervised + + +@pytest.mark.skipif( + not (shutil.which("sandbox-exec") or shutil.which("bwrap")), + reason="requires a real supported sandbox", +) +def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: + marker = tmp_path / "escaped" + child = ( + "import os,sys,time; os.setsid(); time.sleep(1); " + "open(sys.argv[1], 'w').write('escaped')" + ) + parent = ( + "import subprocess,sys; " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " + "start_new_session=False)" + ) + result, surviving = run_supervised( + [sys.executable, "-c", parent, child, str(marker)], + cwd=tmp_path, + timeout=10, + env=dict(os.environ), + writable_roots=(tmp_path,), + ) + assert result.returncode == 0 + assert surviving is True + time.sleep(1.2) + assert not marker.exists() diff --git a/tests/test_sync_determine_operation.py b/tests/test_sync_determine_operation.py index 58d243f10..92b1bb6df 100644 --- a/tests/test_sync_determine_operation.py +++ b/tests/test_sync_determine_operation.py @@ -4187,6 +4187,20 @@ def test_calculate_prompt_hash_with_stored_deps(self, pdd_test_environment): "stored deps should contribute to the composite hash" ) + def test_legacy_include_hash_matches_pre_versioned_bytes_across_cwds( + self, pdd_test_environment, monkeypatch + ): + prompt = pdd_test_environment / "prompts" / f"{BASENAME}_{LANGUAGE}.prompt" + dependency = pdd_test_environment / "shared.py" + create_file(dependency, "VALUE = 1\n") + create_file(prompt, "Build it.\nshared.py\n") + expected = hashlib.sha256(prompt.read_bytes() + dependency.read_bytes()).hexdigest() + hashes = [] + for cwd in (pdd_test_environment, pdd_test_environment.parent): + monkeypatch.chdir(cwd) + hashes.append(calculate_prompt_hash(prompt)) + assert hashes == [expected, expected] + def test_calculate_prompt_hash_detects_dep_change_via_stored_deps(self, pdd_test_environment): """When a stored dep file changes, the composite hash must change.""" prompts_dir = pdd_test_environment / "prompts" From 541fe7bc1108c455bb92ade5620383cd86bf8e88 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 11:40:57 -0700 Subject: [PATCH 052/128] fix(sync): close round 11 verification boundaries --- .github/workflows/unit-tests.yml | 7 +++ pdd/continuous_sync.py | 13 +++- pdd/sync_core/certificate.py | 93 ++++++++++++++++++++++++++++- pdd/sync_core/finalize.py | 6 ++ pdd/sync_core/includes.py | 21 ++++++- pdd/sync_core/lifecycle.py | 6 +- pdd/sync_core/runner.py | 23 ++++++- pdd/sync_core/scenario_harness.py | 6 +- pdd/sync_core/supervisor.py | 80 ++++++++++++++++++++++--- pdd/sync_determine_operation.py | 59 ++++++++++++------ tests/test_sync_core_certificate.py | 7 ++- tests/test_sync_core_runner.py | 6 ++ tests/test_sync_core_supervisor.py | 2 +- 13 files changed, 287 insertions(+), 42 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 98a8134e5..767bd9131 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -52,6 +52,13 @@ jobs: git config --global user.name "PDD CI" git config --global init.defaultBranch main + - name: Provision and verify protected Linux sandbox + run: | + sudo apt-get update + sudo apt-get install --yes bubblewrap + bwrap --unshare-all --die-with-parent --new-session \ + --ro-bind / / --dev /dev --proc /proc -- /bin/true + - name: Install dependencies run: pip install -e ".[dev]" pytest-timeout diff --git a/pdd/continuous_sync.py b/pdd/continuous_sync.py index 7f0a60333..23b9126e3 100644 --- a/pdd/continuous_sync.py +++ b/pdd/continuous_sync.py @@ -110,6 +110,8 @@ def canonical_sync_enabled(root: Path) -> bool: check=False, ) if identity.returncode != 0: + if os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: + raise ValueError("explicit protected sync identity cannot be resolved") return False policy = subprocess.run( ["git", "show", f"{protected_ref}:.pdd/sync-policy.json"], @@ -119,12 +121,19 @@ def canonical_sync_enabled(root: Path) -> bool: check=False, ) if policy.returncode != 0: + if os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: + raise ValueError("explicit protected sync policy cannot be resolved") return False try: payload = json.loads(policy.stdout) - except json.JSONDecodeError: + except json.JSONDecodeError as exc: + if os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: + raise ValueError("explicit protected sync policy is malformed") from exc return False - return payload == {"schema_version": 1, "enforcement": "active"} + active = payload == {"schema_version": 1, "enforcement": "active"} + if not active and os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: + raise ValueError("explicit protected sync policy is not active") + return active def _prompts_dir_for(prompt_path: Path) -> Path: diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index 949065d6d..157b2b523 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -5,6 +5,7 @@ import base64 import ast +import hashlib import json import os import subprocess @@ -394,7 +395,11 @@ def _complete_nightly( row: Any, policy: _NightlyVerificationPolicy, ) -> bool: - if not isinstance(row, dict) or row.get("scan_ok") is not True: + if ( + not isinstance(row, dict) + or row.get("schema_version") != 4 + or row.get("scan_ok") is not True + ): return False repositories = row.get("repositories") counts = row.get("counts") @@ -474,7 +479,11 @@ def _nightly_candidate_artifact_valid( ) except (CandidateArtifactProvenanceError, ValueError): return False - return artifact.wheel_sha256 == lifecycle.get("candidate_wheel_sha256") + if artifact.wheel_sha256 != lifecycle.get("candidate_wheel_sha256"): + return False + return _measurement_payload_bound( + lifecycle, policy.candidate_artifact_policy, artifact + ) def _nightly_observation_complete(payload: Any) -> bool: @@ -618,6 +627,57 @@ def _lifecycle_measurement_complete(lifecycle: LifecycleResult) -> bool: ) +def _installed_closure_digest(installed_files: Any) -> str | None: + """Return the canonical checker-owned installed-file closure digest.""" + if not isinstance(installed_files, (list, tuple)) or not installed_files: + return None + normalized = [list(item) for item in installed_files] + return hashlib.sha256( + json.dumps(normalized, separators=(",", ":")).encode() + ).hexdigest() + + +def _measurement_payload_bound( + lifecycle: dict[str, Any], + policy: CandidateArtifactPolicy, + artifact: CandidateArtifactProvenance, +) -> bool: + """Cross-bind every runtime measurement to policy and provenance.""" + expected_interpreter = { + "implementation": policy.python_implementation, + "version": policy.python_version, + "abi": policy.python_abi, + "platform": policy.python_platform, + } + digest = _installed_closure_digest(lifecycle.get("installed_files")) + return ( + digest is not None + and lifecycle.get("dependency_environment_digest") == digest + and lifecycle.get("runtime_lock_sha256") == policy.dependency_lock_sha256 + and lifecycle.get("interpreter") == expected_interpreter + and lifecycle.get("interpreter") == artifact.python + and artifact.dependency_lock_sha256 == policy.dependency_lock_sha256 + ) + + +def _lifecycle_measurement_bound( + lifecycle: LifecycleResult, + policy: CandidateArtifactPolicy, +) -> bool: + if lifecycle.candidate_artifact is None: + return False + return _measurement_payload_bound( + { + "dependency_environment_digest": lifecycle.dependency_environment_digest, + "runtime_lock_sha256": lifecycle.runtime_lock_sha256, + "interpreter": lifecycle.interpreter, + "installed_files": lifecycle.installed_files, + }, + policy, + lifecycle.candidate_artifact, + ) + + def _predicate( counts: dict[str, int], lifecycle: LifecycleResult, extra: dict[str, int], *, require_measurement: bool = True, @@ -688,6 +748,7 @@ def _canonical_report_predicate(report: dict[str, Any]) -> bool: def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool]: # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches + # pylint: disable=too-many-statements schema_version = body.get("schema_version") if schema_version not in {3, 4}: return False, False @@ -785,6 +846,26 @@ def _recompute_certificate_predicates(body: dict[str, Any]) -> tuple[bool, bool] return False, False if candidate_artifact.wheel_sha256 != digests["candidate_wheel_sha256"]: return False, False + policy_payload = body.get("candidate_artifact_policy") + if not isinstance(policy_payload, dict): + return False, False + try: + embedded_policy = CandidateArtifactPolicy( + str(policy_payload.get("issuer", "")), + b"0" * 32, + str(policy_payload.get("builder_workflow_identity", "")), + str(policy_payload.get("dependency_lock_sha256", "")), + str(policy_payload.get("python_implementation", "")), + str(policy_payload.get("python_version", "")), + str(policy_payload.get("python_abi", "")), + str(policy_payload.get("python_platform", "")), + ) + except (CandidateArtifactProvenanceError, ValueError): + return False, False + if schema_version == 4 and not _measurement_payload_bound( + lifecycle_payload, embedded_policy, candidate_artifact + ): + return False, False pdd_report = next( ( item @@ -917,7 +998,9 @@ def _build_global_certificate_from_targets( } lifecycle = ( options.lifecycle_result - if candidate_artifact_valid + if candidate_artifact_valid and _lifecycle_measurement_bound( + options.lifecycle_result, options.candidate_artifact_policy + ) else replace(options.lifecycle_result, candidate_artifact=None) ) body: dict[str, Any] = { @@ -1083,6 +1166,10 @@ def verify_global_certificate( ) except (CandidateArtifactProvenanceError, AttributeError, ValueError): return False + if not _measurement_payload_bound( + lifecycle, expected_candidate_artifact_policy, artifact + ): + return False try: checked_at = datetime.fromisoformat(str(certificate["checked_at"])) except (KeyError, ValueError): diff --git a/pdd/sync_core/finalize.py b/pdd/sync_core/finalize.py index 327cf0c54..e2f9da4b4 100644 --- a/pdd/sync_core/finalize.py +++ b/pdd/sync_core/finalize.py @@ -264,6 +264,12 @@ def finalize_unit( ) if reusable is not None: return reusable + cleanliness = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=repository_root, capture_output=True, text=True, check=False, + ) + if cleanliness.returncode != 0 or cleanliness.stdout: + raise ValueError("canonical finalization requires a completely clean checkout") transaction_id = f"finalize-{uuid.uuid4()}" attestation_id = f"attestation-{uuid.uuid4()}" envelope, executions = run_profile( diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py index 57f1526f4..e3c6667d3 100644 --- a/pdd/sync_core/includes.py +++ b/pdd/sync_core/includes.py @@ -103,12 +103,31 @@ def _is_attribute_name_char(character: str) -> bool: def _has_boolean_attr(raw: str, name: str) -> bool: - """Find one bare boolean attribute with the legacy ASCII boundaries.""" + """Find one bare boolean attribute outside quoted attribute values.""" cursor = 0 + quote: str | None = None while True: + if cursor >= len(raw): + return False + character = raw[cursor] + if quote is not None: + if character == quote: + quote = None + cursor += 1 + continue + if character in "\"'": + quote = character + cursor += 1 + continue start = raw.find(name, cursor) if start < 0: return False + intervening = raw[cursor:start] + quote_positions = [position for position in ( + intervening.find("\""), intervening.find("'")) if position >= 0] + if quote_positions: + cursor += min(quote_positions) + continue end = start + len(name) before_is_word = start > 0 and ( raw[start - 1].isascii() and _is_attribute_name_char(raw[start - 1]) diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 41588199b..759d4442b 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -294,7 +294,6 @@ def run_lifecycle_matrix( ) except (OSError, CandidateArtifactProvenanceError): return _failed_result() - output = temporary / "result.json" (temporary / "home").mkdir(mode=0o700) installed_candidate = _install_candidate_wheel( temporary, @@ -323,7 +322,7 @@ def run_lifecycle_matrix( "-m", "pdd.sync_core.scenario_harness", "--output", - str(output), + "-", "--cloud-root", str(Path(cloud_root).resolve()), "--cloud-base-ref", @@ -339,7 +338,8 @@ def run_lifecycle_matrix( if completed.returncode == 124: return _failed_result(timeout=True) try: - results = _normalized_results(json.loads(output.read_text(encoding="utf-8"))) + lines = [line for line in completed.stdout.splitlines() if line.strip()] + results = _normalized_results(json.loads(lines[-1])) except (OSError, ValueError, json.JSONDecodeError): return _failed_result() missing = tuple( diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 7c0767bb6..a3c406762 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -305,7 +305,8 @@ def pytest_validator_config_digest( def _has_dynamic_pytest_plugins( - root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...], + code_under_test_paths: tuple[PurePosixPath, ...] = (), ) -> bool: """Fail closed when pytest plugin declarations cannot be statically bound.""" config_paths = _pytest_config_paths(root, ref, test_paths) @@ -324,6 +325,7 @@ def _has_dynamic_pytest_plugins( ref, path, source, + code_under_test_paths=frozenset(code_under_test_paths), ) if dynamic: return True @@ -910,8 +912,10 @@ def _obligation_preflight( obligation.validator_config_digest, "declared validator config digest does not match protected closure", ) - if _has_dynamic_pytest_plugins(root, base_sha, obligation.artifact_paths) or ( - _has_dynamic_pytest_plugins(root, head_sha, obligation.artifact_paths) + if _has_dynamic_pytest_plugins(root, base_sha, obligation.artifact_paths, + obligation.code_under_test_paths) or ( + _has_dynamic_pytest_plugins(root, head_sha, obligation.artifact_paths, + obligation.code_under_test_paths) ): return RunnerExecution( obligation.obligation_id, @@ -1032,6 +1036,19 @@ def run_obligation( config: RunnerConfig, ) -> RunnerExecution: """Run an obligation in an ephemeral exact-head execution tree.""" + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], cwd=root, + capture_output=True, text=True, check=False, + ) + dirty_all = tuple(line[3:] for line in status.stdout.splitlines() if len(line) >= 4) + if status.returncode != 0 or dirty_all: + return RunnerExecution( + obligation.obligation_id, + EvidenceOutcome.QUARANTINED, + obligation.validator_config_digest, + "dirty checkout cannot receive committed-head evidence: " + + ", ".join(dirty_all), + ) dirty = _dirty_pytest_support(root) if dirty: return RunnerExecution( diff --git a/pdd/sync_core/scenario_harness.py b/pdd/sync_core/scenario_harness.py index 662ee0fa1..d45bbbd5f 100644 --- a/pdd/sync_core/scenario_harness.py +++ b/pdd/sync_core/scenario_harness.py @@ -860,7 +860,11 @@ def main() -> None: "schema_version": 1, "results": [asdict(result) for result in results], } - arguments.output.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + encoded = json.dumps(payload, sort_keys=True) + if str(arguments.output) == "-": + print(encoded, flush=True) + else: + arguments.output.write_text(encoded, encoding="utf-8") if any(result.status != "PASS" for result in results): raise SystemExit(1) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 16c03c14d..537cf38f6 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -8,9 +8,40 @@ import subprocess import sys import tempfile +import time +import uuid from pathlib import Path +def _supervised_descendants(token: str) -> set[int]: + """Find descendants carrying the unforgeable per-run environment marker.""" + found: set[int] = set() + if sys.platform.startswith("linux"): + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + environment = (entry / "environ").read_bytes() + except (OSError, PermissionError): + continue + if f"PDD_SUPERVISION_TOKEN={token}".encode() in environment.split(b"\0"): + found.add(int(entry.name)) + return found + listing = subprocess.run( + ["ps", "eww", "-axo", "pid=,command="], capture_output=True, + text=True, check=False, + ) + marker = f"PDD_SUPERVISION_TOKEN={token}" + for line in listing.stdout.splitlines(): + if marker not in line: + continue + try: + found.add(int(line.strip().split(None, 1)[0])) + except (ValueError, IndexError): + continue + return found + + def _sandbox_command( command: list[str], writable_roots: tuple[Path, ...] ) -> tuple[list[str], Path | None]: @@ -39,28 +70,53 @@ def run_supervised( command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], writable_roots: tuple[Path, ...], ) -> tuple[subprocess.CompletedProcess[str], bool]: - """Run inside a supported networkless sandbox and kill the whole process group.""" - # pylint: disable=consider-using-with + """Run sandboxed and terminate marked descendants across session changes.""" + # pylint: disable=consider-using-with,too-many-locals,too-many-branches try: argv, profile = _sandbox_command(command, writable_roots) except RuntimeError as exc: return subprocess.CompletedProcess(command, 125, "", str(exc)), False + token = uuid.uuid4().hex process = subprocess.Popen( argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env | {"PYTHONDONTWRITEBYTECODE": "1", + "PDD_SUPERVISION_TOKEN": token, "TMPDIR": str(writable_roots[0].resolve()), "TEMP": str(writable_roots[0].resolve()), "TMP": str(writable_roots[0].resolve())}, start_new_session=True, ) timed_out = False - try: - stdout, stderr = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - timed_out = True - os.killpg(process.pid, signal.SIGKILL) - stdout, stderr = process.communicate() surviving = False + deadline = time.monotonic() + timeout + while True: + try: + stdout, stderr = process.communicate( + timeout=max(0.01, min(0.1, deadline - time.monotonic())) + ) + break + except subprocess.TimeoutExpired: + descendants = _supervised_descendants(token) - {process.pid} + if process.poll() is not None and descendants: + surviving = True + for pid in descendants: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if time.monotonic() >= deadline: + timed_out = True + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for pid in _supervised_descendants(token) - {process.pid}: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + break if not timed_out and os.name != "nt": try: os.killpg(process.pid, 0) @@ -68,6 +124,14 @@ def run_supervised( os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass + descendants = _supervised_descendants(token) - {process.pid} + if descendants: + surviving = True + for pid in descendants: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass if profile is not None: profile.unlink(missing_ok=True) return subprocess.CompletedProcess(command, 124 if timed_out else process.returncode, diff --git a/pdd/sync_determine_operation.py b/pdd/sync_determine_operation.py index 75a4ec51a..ad541dcb6 100644 --- a/pdd/sync_determine_operation.py +++ b/pdd/sync_determine_operation.py @@ -1779,7 +1779,24 @@ def extract_include_deps(prompt_path: Path) -> Dict[str, str]: return {item.relpath.as_posix(): item.digest for item in closure.artifacts} -def calculate_prompt_hash(prompt_path: Path, stored_deps: Optional[Dict[str, str]] = None) -> Optional[str]: +def _legacy_dependency_path(prompt_path: Path, declared: str) -> Optional[Path]: + """Resolve a legacy include from prompt ancestry, never the process CWD.""" + candidate = Path(declared) + if candidate.is_absolute(): + return candidate if candidate.is_file() else None + for parent in (prompt_path.parent, *prompt_path.parent.parents): + resolved = parent / candidate + if resolved.is_file(): + return resolved + return None + + +def calculate_prompt_hash( + prompt_path: Path, + stored_deps: Optional[Dict[str, str]] = None, + *, + hash_version: int = 1, +) -> Optional[str]: """Hash a prompt file including the content of all its dependencies. If the prompt has tags, extracts and hashes those dependencies. @@ -1804,27 +1821,33 @@ def calculate_prompt_hash(prompt_path: Path, stored_deps: Optional[Dict[str, str from pdd.sync_core.includes import parse_include_references references = parse_include_references(prompt_content) - if references: - try: - dependencies = extract_include_deps(prompt_path) - except (FileNotFoundError, ValueError): + declared_dependencies = ( + [reference.path for reference in references] + if references else list((stored_deps or {}).keys()) + ) + resolved_dependencies = [] + for declared in declared_dependencies: + candidate = _legacy_dependency_path(prompt_path.resolve(), declared) + if candidate is None: return None - else: - dependencies = dict(stored_deps or {}) - for path, expected_digest in dependencies.items(): - candidate = Path(path) - if not candidate.is_absolute(): - candidate = Path.cwd() / candidate - if calculate_sha256(candidate) != expected_digest: - if not candidate.is_file(): - return None - dependencies[path] = calculate_sha256(candidate) or "" + resolved_dependencies.append(candidate.resolve()) hasher = hashlib.sha256() hasher.update(prompt_path.read_bytes()) - for path, digest in sorted(dependencies.items()): - hasher.update(path.encode("utf-8")) - hasher.update(digest.encode("ascii")) + if hash_version == 1: + for dependency in resolved_dependencies: + hasher.update(dependency.read_bytes()) + elif hash_version == 2: + anchor = prompt_path.resolve().parent + for dependency in sorted(resolved_dependencies): + try: + key = dependency.relative_to(anchor).as_posix() + except ValueError: + key = dependency.as_posix() + hasher.update(key.encode("utf-8") + b"\0") + hasher.update(bytes.fromhex(calculate_sha256(dependency) or "")) + else: + raise ValueError(f"unsupported prompt hash version: {hash_version}") return hasher.hexdigest() diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 91e93de79..cd577b03c 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -37,7 +37,10 @@ ) OBSERVATION = NightlyObservation(True, True, 0, True, "BLOCKED", 0) CANDIDATE_WHEEL_SHA256 = "d" * 64 -DEPENDENCY_ENVIRONMENT_DIGEST = "e" * 64 +INSTALLED_FILES = (("pdd", "1.0", "pdd/__init__.py", "f" * 64),) +DEPENDENCY_ENVIRONMENT_DIGEST = hashlib.sha256( + json.dumps(INSTALLED_FILES, separators=(",", ":")).encode() +).hexdigest() CANDIDATE_BUILDER = AttestationSigner("candidate-builder", b"h" * 32) CANDIDATE_POLICY = CandidateArtifactPolicy( CANDIDATE_BUILDER.issuer, @@ -53,7 +56,7 @@ "runtime_lock_sha256": DEPENDENCY_ENVIRONMENT_DIGEST, "interpreter": {"implementation": "CPython", "version": "3.12.3", "abi": "cp312", "platform": "manylinux_2_17_x86_64"}, - "installed_files": (("pdd", "1.0", "pdd/__init__.py", "f" * 64),), + "installed_files": INSTALLED_FILES, "measurement_authority": "pdd-released-checker-v1", } diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index ca1d99414..68ddf64be 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -508,6 +508,9 @@ def test_collection_probe_is_checker_owned_not_candidate_shadow(tmp_path) -> Non " items[:] = []\n", encoding="utf-8", ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate probe shadow") + head = _git(root, "rev-parse", "HEAD") _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.PASS assert not (root / "candidate-probe-loaded").exists() @@ -525,6 +528,9 @@ def test_collection_probe_fixed_name_is_not_candidate_shadowable(tmp_path) -> No " Path(output).write_text(json.dumps([item.nodeid for item in items]))\n", encoding="utf-8", ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "candidate fixed probe shadow") + head = _git(root, "rev-parse", "HEAD") _envelope, executions = _run(root, head, head) assert executions[0].outcome is EvidenceOutcome.PASS assert not (root / "candidate-fixed-probe-loaded").exists() diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 36997ab2c..9a1861dc8 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -18,7 +18,7 @@ def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: marker = tmp_path / "escaped" child = ( - "import os,sys,time; os.setsid(); time.sleep(1); " + "import os,sys,time; os.setsid(); os.close(1); os.close(2); time.sleep(1); " "open(sys.argv[1], 'w').write('escaped')" ) parent = ( From a90586613cee2d755c7ebd673c9084e5525fa8f8 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 12:19:04 -0700 Subject: [PATCH 053/128] test(sync): expose round 13 verification boundaries --- tests/test_sync_core_reporting.py | 15 ++++++++++ tests/test_sync_core_runner.py | 16 +++++++++++ tests/test_sync_core_snapshot.py | 19 +++++++++++++ tests/test_sync_core_supervisor.py | 38 +++++++++++++++++++++++++- tests/test_sync_determine_operation.py | 18 ++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index 9cd118ef4..6e99cd701 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -524,6 +524,21 @@ def test_trusted_finalizer_second_run_is_zero_write_no_op(tmp_path) -> None: path.write_text(json.dumps(payload, sort_keys=True)) +def test_trusted_finalizer_rejects_dirty_support_before_reuse(tmp_path) -> None: + root, commit = _repository(tmp_path) + replay = tmp_path / "external-trust/dirty-reuse.json" + finalize_unit( + root, PurePosixPath("prompts/widget_python.prompt"), base_ref=commit, + head_ref=commit, signer=SIGNER, replay_ledger_path=replay, + ) + (root / "conftest.py").write_text("pytest_plugins = []\n") + with pytest.raises(ValueError, match="completely clean checkout"): + finalize_unit( + root, PurePosixPath("prompts/widget_python.prompt"), base_ref=commit, + head_ref=commit, signer=SIGNER, replay_ledger_path=replay, + ) + + @pytest.mark.parametrize( ("edits", "semantic", "changed_roles"), [ diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 68ddf64be..e727c2beb 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -296,6 +296,22 @@ def test_nested_external_pytest_plugin_fails_closed(tmp_path) -> None: assert "external pytest_plugins" in executions[0].detail +def test_product_pytest_plugins_declaration_is_not_traversed_as_support(tmp_path) -> None: + root, _base = _repository( + tmp_path, "import product\ndef test_widget(): assert product.VALUE == 1\n" + ) + (root / "product.py").write_text( + 'VALUE = 1\npytest_plugins = "vendor.runtime"\n' + ) + _git(root, "add", "product.py", "tests/test_widget.py") + _git(root, "commit", "-q", "-m", "declared product") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run( + root, head, head, (PurePosixPath("product.py"),) + ) + assert executions[0].outcome is EvidenceOutcome.PASS + + def test_dynamic_repo_local_import_fails_closed(tmp_path) -> None: root, _base = _repository(tmp_path, "def test_widget(): assert True\n") (root / "tests/test_widget.py").write_text( diff --git a/tests/test_sync_core_snapshot.py b/tests/test_sync_core_snapshot.py index fcd66b490..340890f1f 100644 --- a/tests/test_sync_core_snapshot.py +++ b/tests/test_sync_core_snapshot.py @@ -81,6 +81,7 @@ def _repository(tmp_path: Path, *, query: bool = False) -> tuple[Path, str]: "tests/test_widget.py", "tests/test_widget_e2e.py", ], + "code_under_test_paths": ["notes.md"], } ], } @@ -105,6 +106,7 @@ def test_snapshot_contains_prompt_include_code_and_all_tests(tmp_path) -> None: ("code", "src/widget.py"), ("test", "tests/test_widget.py"), ("test", "tests/test_widget_e2e.py"), + ("code", "notes.md"), } executable = next( item for item in snapshot.artifacts if item.relpath.name == "test_widget_e2e.py" @@ -112,6 +114,23 @@ def test_snapshot_contains_prompt_include_code_and_all_tests(tmp_path) -> None: assert executable.git_mode == "100755" +def test_code_under_test_bytes_invalidate_snapshot(tmp_path) -> None: + root, base = _repository(tmp_path) + manifest = build_unit_manifest(root, base_ref=base, head_ref=base) + profile = load_verification_profiles(root, manifest).profiles[0] + before = build_unit_snapshot(root, manifest, manifest.managed_units[0], profile) + (root / "notes.md").write_text("Changed product bytes\n") + _git(root, "add", "notes.md") + _git(root, "commit", "-q", "-m", "change declared product") + head = _git(root, "rev-parse", "HEAD") + changed_manifest = build_unit_manifest(root, base_ref=base, head_ref=head) + changed_profile = load_verification_profiles(root, changed_manifest).profiles[0] + after = build_unit_snapshot( + root, changed_manifest, changed_manifest.managed_units[0], changed_profile + ) + assert before.digest() != after.digest() + + def test_query_expansion_cannot_receive_trusted_verified_status(tmp_path) -> None: root, commit = _repository(tmp_path, query=True) manifest = build_unit_manifest(root, base_ref=commit, head_ref=commit) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 9a1861dc8..dcfc7b1e4 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -8,7 +8,18 @@ import pytest -from pdd.sync_core.supervisor import run_supervised +from pdd.sync_core.supervisor import _sandbox_command, run_supervised + + +def test_linux_sandbox_uses_network_namespace_without_loopback_setup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + argv, profile = _sandbox_command(["/bin/true"], (tmp_path,)) + assert profile is None + assert argv[:5] == ["unshare", "--user", "--map-root-user", "--net", "--"] + assert "--unshare-net" not in argv @pytest.mark.skipif( @@ -37,3 +48,28 @@ def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: assert surviving is True time.sleep(1.2) assert not marker.exists() + + +@pytest.mark.skipif( + not (shutil.which("sandbox-exec") or shutil.which("bwrap")), + reason="requires a real supported sandbox", +) +def test_detached_descendant_cannot_escape_by_removing_marker(tmp_path: Path) -> None: + marker = tmp_path / "escaped-without-marker" + child = ( + "import os,sys,time; os.environ.pop('PDD_SUPERVISION_TOKEN', None); " + "os.setsid(); os.close(1); os.close(2); time.sleep(1); " + "open(sys.argv[1], 'w').write('escaped')" + ) + parent = ( + "import subprocess,sys; " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]])" + ) + result, surviving = run_supervised( + [sys.executable, "-c", parent, child, str(marker)], cwd=tmp_path, + timeout=10, env=dict(os.environ), writable_roots=(tmp_path,), + ) + assert result.returncode == 0 + assert surviving is True + time.sleep(1.2) + assert not marker.exists() diff --git a/tests/test_sync_determine_operation.py b/tests/test_sync_determine_operation.py index 92b1bb6df..5938ec4b4 100644 --- a/tests/test_sync_determine_operation.py +++ b/tests/test_sync_determine_operation.py @@ -4201,6 +4201,24 @@ def test_legacy_include_hash_matches_pre_versioned_bytes_across_cwds( hashes.append(calculate_prompt_hash(prompt)) assert hashes == [expected, expected] + def test_legacy_include_hash_sorts_and_deduplicates_dependencies( + self, pdd_test_environment + ): + prompt = pdd_test_environment / "prompts" / f"{BASENAME}_{LANGUAGE}.prompt" + first = pdd_test_environment / "a.py" + second = pdd_test_environment / "b.py" + create_file(first, "A = 1\n") + create_file(second, "B = 1\n") + create_file( + prompt, + "Build.\nb.py\na.py\n" + "a.py\n", + ) + expected = hashlib.sha256( + prompt.read_bytes() + first.read_bytes() + second.read_bytes() + ).hexdigest() + assert calculate_prompt_hash(prompt, hash_version=1) == expected + def test_calculate_prompt_hash_detects_dep_change_via_stored_deps(self, pdd_test_environment): """When a stored dep file changes, the composite hash must change.""" prompts_dir = pdd_test_environment / "prompts" From b61046f4a8b581cab69ea367b0b0be87d07bc4c3 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 12:26:37 -0700 Subject: [PATCH 054/128] fix(sync): close round 13 verification boundaries --- .github/workflows/unit-tests.yml | 6 ++-- pdd/sync_core/finalize.py | 26 +++++++++++---- pdd/sync_core/runner.py | 14 +++++--- pdd/sync_core/snapshot.py | 2 ++ pdd/sync_core/supervisor.py | 51 +++++++++++++++++++++++++++--- pdd/sync_determine_operation.py | 2 ++ tests/test_sync_core_snapshot.py | 7 ++-- tests/test_sync_core_supervisor.py | 9 +++--- 8 files changed, 93 insertions(+), 24 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 767bd9131..f634996ed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -56,8 +56,10 @@ jobs: run: | sudo apt-get update sudo apt-get install --yes bubblewrap - bwrap --unshare-all --die-with-parent --new-session \ - --ro-bind / / --dev /dev --proc /proc -- /bin/true + unshare --user --map-root-user --net -- \ + bwrap --unshare-pid --unshare-ipc --unshare-uts \ + --die-with-parent --new-session --ro-bind / / \ + --dev /dev --proc /proc -- /bin/true - name: Install dependencies run: pip install -e ".[dev]" pytest-timeout diff --git a/pdd/sync_core/finalize.py b/pdd/sync_core/finalize.py index e2f9da4b4..a8a2512f9 100644 --- a/pdd/sync_core/finalize.py +++ b/pdd/sync_core/finalize.py @@ -214,6 +214,24 @@ def _reusable_result( return FinalizeResult(transaction, envelope.attestation_id, fingerprint) +def _checkout_is_clean_for_finalization(root: Path) -> bool: + """Allow only checker-owned durable state from an earlier finalization.""" + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=root, capture_output=True, text=True, check=False, + ) + if status.returncode != 0: + return False + allowed = ( + ".pdd/meta/v2/", ".pdd/evidence/v2/", ".pdd/locks/transactions/", + ".pdd/transactions/", + ) + return all( + len(line) >= 4 and line[3:].replace('"', "").startswith(allowed) + for line in status.stdout.splitlines() + ) + + def finalize_unit( root: Path, module: PurePosixPath, @@ -252,6 +270,8 @@ def finalize_unit( verifier = load_trust_policy( repository_root, base_sha, replay_ledger_path=replay ).verifier + if not _checkout_is_clean_for_finalization(repository_root): + raise ValueError("canonical finalization requires a completely clean checkout") reusable = _reusable_result( repository_root, snapshot, @@ -264,12 +284,6 @@ def finalize_unit( ) if reusable is not None: return reusable - cleanliness = subprocess.run( - ["git", "status", "--porcelain", "--untracked-files=all"], - cwd=repository_root, capture_output=True, text=True, check=False, - ) - if cleanliness.returncode != 0 or cleanliness.stdout: - raise ValueError("canonical finalization requires a completely clean checkout") transaction_id = f"finalize-{uuid.uuid4()}" attestation_id = f"attestation-{uuid.uuid4()}" envelope, executions = run_profile( diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index a3c406762..e0f7ed089 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -334,7 +334,8 @@ def _has_dynamic_pytest_plugins( def _has_external_pytest_plugins( - root: Path, ref: str, test_paths: tuple[PurePosixPath, ...] + root: Path, ref: str, test_paths: tuple[PurePosixPath, ...], + code_under_test_paths: tuple[PurePosixPath, ...] = (), ) -> bool: """Return whether a literal plugin declaration lacks protected repo bytes.""" remaining = list(tuple(test_paths) + tuple(_pytest_config_paths(root, ref, test_paths))) @@ -365,7 +366,10 @@ def _has_external_pytest_plugins( ) if not any(read_git_blob(root, ref, item) is not None for item in candidates): return True - discovered, _dynamic = _local_module_paths(root, ref, path, source) + discovered, _dynamic = _local_module_paths( + root, ref, path, source, + code_under_test_paths=frozenset(code_under_test_paths), + ) remaining.extend(discovered - visited) return False @@ -923,8 +927,10 @@ def _obligation_preflight( obligation.validator_config_digest, "dynamic pytest_plugins declarations are not bound by this adapter", ) - if _has_external_pytest_plugins(root, base_sha, obligation.artifact_paths) or ( - _has_external_pytest_plugins(root, head_sha, obligation.artifact_paths) + if _has_external_pytest_plugins( + root, base_sha, obligation.artifact_paths, obligation.code_under_test_paths + ) or _has_external_pytest_plugins( + root, head_sha, obligation.artifact_paths, obligation.code_under_test_paths ): return RunnerExecution( obligation.obligation_id, diff --git a/pdd/sync_core/snapshot.py b/pdd/sync_core/snapshot.py index c26453999..619c6cd96 100644 --- a/pdd/sync_core/snapshot.py +++ b/pdd/sync_core/snapshot.py @@ -53,6 +53,8 @@ def add(role: str, relpath: PurePosixPath, required: bool = True) -> None: for path in unit.artifact_paths: add("code", path) for obligation in profile.obligations: + for path in obligation.code_under_test_paths: + add("code", path) role = _obligation_role(obligation.kind) for path in obligation.artifact_paths: add(role, path, required=obligation.required) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 537cf38f6..8258fa841 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile +import threading import time import uuid from pathlib import Path @@ -42,6 +43,29 @@ def _supervised_descendants(token: str) -> set[int]: return found +def _process_descendants(root_pid: int) -> set[int]: + """Return the current transitive process tree without trusting child state.""" + listing = subprocess.run( + ["ps", "-axo", "pid=,ppid="], capture_output=True, text=True, check=False, + ) + children: dict[int, set[int]] = {} + for line in listing.stdout.splitlines(): + try: + pid, parent = (int(value) for value in line.split()) + except (ValueError, TypeError): + continue + children.setdefault(parent, set()).add(pid) + found: set[int] = set() + pending = [root_pid] + while pending: + parent = pending.pop() + for child in children.get(parent, ()): + if child not in found: + found.add(child) + pending.append(child) + return found + + def _sandbox_command( command: list[str], writable_roots: tuple[Path, ...] ) -> tuple[list[str], Path | None]: @@ -56,9 +80,15 @@ def _sandbox_command( profile = Path(name) profile.write_text("\n".join(rules), encoding="utf-8") return ["sandbox-exec", "-f", str(profile), *command], profile - if sys.platform.startswith("linux") and shutil.which("bwrap"): - argv = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session", - "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc"] + if (sys.platform.startswith("linux") and shutil.which("bwrap") + and shutil.which("unshare")): + # util-linux creates the network namespace but does not configure its + # loopback device, avoiding the RTM_NEWADDR operation denied by hosted + # GitHub runners. Bubblewrap supplies the mount and PID containment. + argv = ["unshare", "--user", "--map-root-user", "--net", "--", + "bwrap", "--unshare-pid", "--unshare-ipc", "--unshare-uts", + "--die-with-parent", "--new-session", "--ro-bind", "/", "/", + "--dev", "/dev", "--proc", "/proc"] for item in writable_roots: resolved = str(item.resolve()) argv.extend(("--bind", resolved, resolved)) @@ -71,7 +101,7 @@ def run_supervised( writable_roots: tuple[Path, ...], ) -> tuple[subprocess.CompletedProcess[str], bool]: """Run sandboxed and terminate marked descendants across session changes.""" - # pylint: disable=consider-using-with,too-many-locals,too-many-branches + # pylint: disable=consider-using-with,too-many-locals,too-many-branches,too-many-statements try: argv, profile = _sandbox_command(command, writable_roots) except RuntimeError as exc: @@ -88,6 +118,15 @@ def run_supervised( ) timed_out = False surviving = False + tracked: set[int] = set() + tracking_done = threading.Event() + + def track_process_tree() -> None: + while not tracking_done.wait(0.005): + tracked.update(_process_descendants(process.pid)) + + tracker = threading.Thread(target=track_process_tree, daemon=True) + tracker.start() deadline = time.monotonic() + timeout while True: try: @@ -117,6 +156,8 @@ def run_supervised( pass stdout, stderr = process.communicate() break + tracking_done.set() + tracker.join(timeout=1) if not timed_out and os.name != "nt": try: os.killpg(process.pid, 0) @@ -124,7 +165,7 @@ def run_supervised( os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass - descendants = _supervised_descendants(token) - {process.pid} + descendants = (_supervised_descendants(token) | tracked) - {process.pid} if descendants: surviving = True for pid in descendants: diff --git a/pdd/sync_determine_operation.py b/pdd/sync_determine_operation.py index ad541dcb6..baaf10eb1 100644 --- a/pdd/sync_determine_operation.py +++ b/pdd/sync_determine_operation.py @@ -1825,6 +1825,8 @@ def calculate_prompt_hash( [reference.path for reference in references] if references else list((stored_deps or {}).keys()) ) + if hash_version == 1: + declared_dependencies = sorted(set(declared_dependencies)) resolved_dependencies = [] for declared in declared_dependencies: candidate = _legacy_dependency_path(prompt_path.resolve(), declared) diff --git a/tests/test_sync_core_snapshot.py b/tests/test_sync_core_snapshot.py index 340890f1f..aa7fac87c 100644 --- a/tests/test_sync_core_snapshot.py +++ b/tests/test_sync_core_snapshot.py @@ -42,12 +42,13 @@ def _repository(tmp_path: Path, *, query: bool = False) -> tuple[Path, str]: (root / "tests/test_widget.py").write_text("def test_widget(): pass\n") (root / "tests/test_widget_e2e.py").write_text("def test_e2e(): pass\n") (root / "notes.md").write_text("Human-owned release notes\n") + (root / "release.md").write_text("Unrelated human-owned file\n") (root / ".pdd/sync-ownership.json").write_text( json.dumps( { "rules": [ { - "pattern": "notes.md", + "pattern": "*.md", "inventory": "HUMAN_OWNED", "role": "documentation", "owner": "docs@example.com", @@ -147,8 +148,8 @@ def test_unrelated_human_owned_change_does_not_invalidate_unit_snapshot(tmp_path root, first_manifest, first_manifest.managed_units[0], first_profile ) - (root / "notes.md").write_text("Unrelated human-owned update\n") - _git(root, "add", "notes.md") + (root / "release.md").write_text("Unrelated human-owned update\n") + _git(root, "add", "release.md") _git(root, "commit", "-q", "-m", "update release notes") head = _git(root, "rev-parse", "HEAD") second_manifest = build_unit_manifest(root, base_ref=base, head_ref=head) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index dcfc7b1e4..4a2f2e4aa 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -57,13 +57,14 @@ def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: def test_detached_descendant_cannot_escape_by_removing_marker(tmp_path: Path) -> None: marker = tmp_path / "escaped-without-marker" child = ( - "import os,sys,time; os.environ.pop('PDD_SUPERVISION_TOKEN', None); " - "os.setsid(); os.close(1); os.close(2); time.sleep(1); " + "import os,sys,time; os.setsid(); os.close(1); os.close(2); time.sleep(1); " "open(sys.argv[1], 'w').write('escaped')" ) parent = ( - "import subprocess,sys; " - "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]])" + "import os,subprocess,sys,time; child_env=dict(os.environ); " + "child_env.pop('PDD_SUPERVISION_TOKEN', None); " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " + "env=child_env); time.sleep(0.1)" ) result, surviving = run_supervised( [sys.executable, "-c", parent, child, str(marker)], cwd=tmp_path, From 48fb2a04c3c415aa8b491796c0f318d9b409ea23 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 12:35:56 -0700 Subject: [PATCH 055/128] fix(sync): ignore reaped supervised processes --- pdd/sync_core/supervisor.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 8258fa841..e221cc976 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -66,6 +66,18 @@ def _process_descendants(root_pid: int) -> set[int]: return found +def _live_processes(pids: set[int]) -> set[int]: + """Filter historical observations to processes that still exist.""" + live = set() + for pid in pids: + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + continue + live.add(pid) + return live + + def _sandbox_command( command: list[str], writable_roots: tuple[Path, ...] ) -> tuple[list[str], Path | None]: @@ -165,7 +177,9 @@ def track_process_tree() -> None: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass - descendants = (_supervised_descendants(token) | tracked) - {process.pid} + descendants = _live_processes( + (_supervised_descendants(token) | tracked) - {process.pid} + ) if descendants: surviving = True for pid in descendants: From c8e8842e9ec985ca9bd454edb41da2ccdff7534f Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 12:38:48 -0700 Subject: [PATCH 056/128] fix(sync): provision hosted runner sandbox with sudo --- .github/workflows/unit-tests.yml | 5 ++--- pdd/sync_core/supervisor.py | 16 ++++++++-------- tests/test_sync_core_supervisor.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f634996ed..28986725f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -56,9 +56,8 @@ jobs: run: | sudo apt-get update sudo apt-get install --yes bubblewrap - unshare --user --map-root-user --net -- \ - bwrap --unshare-pid --unshare-ipc --unshare-uts \ - --die-with-parent --new-session --ro-bind / / \ + sudo -n -E bwrap --unshare-all --die-with-parent --new-session \ + --uid "$(id -u)" --gid "$(id -g)" --ro-bind / / \ --dev /dev --proc /proc -- /bin/true - name: Install dependencies diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index e221cc976..04674de1e 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -92,14 +92,14 @@ def _sandbox_command( profile = Path(name) profile.write_text("\n".join(rules), encoding="utf-8") return ["sandbox-exec", "-f", str(profile), *command], profile - if (sys.platform.startswith("linux") and shutil.which("bwrap") - and shutil.which("unshare")): - # util-linux creates the network namespace but does not configure its - # loopback device, avoiding the RTM_NEWADDR operation denied by hosted - # GitHub runners. Bubblewrap supplies the mount and PID containment. - argv = ["unshare", "--user", "--map-root-user", "--net", "--", - "bwrap", "--unshare-pid", "--unshare-ipc", "--unshare-uts", - "--die-with-parent", "--new-session", "--ro-bind", "/", "/", + if sys.platform.startswith("linux") and shutil.which("bwrap"): + elevated = bool(shutil.which("sudo")) and subprocess.run( + ["sudo", "-n", "true"], capture_output=True, check=False, + ).returncode == 0 + prefix = ["sudo", "-n", "-E"] if elevated else [] + argv = [*prefix, "bwrap", "--unshare-all", "--die-with-parent", + "--new-session", "--uid", str(os.getuid()), "--gid", + str(os.getgid()), "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc"] for item in writable_roots: resolved = str(item.resolve()) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 4a2f2e4aa..3ee4b8b0c 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -2,6 +2,7 @@ import os import shutil +import subprocess import sys import time from pathlib import Path @@ -11,15 +12,20 @@ from pdd.sync_core.supervisor import _sandbox_command, run_supervised -def test_linux_sandbox_uses_network_namespace_without_loopback_setup( +def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(sys, "platform", "linux") monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr( + "pdd.sync_core.supervisor.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""), + ) argv, profile = _sandbox_command(["/bin/true"], (tmp_path,)) assert profile is None - assert argv[:5] == ["unshare", "--user", "--map-root-user", "--net", "--"] - assert "--unshare-net" not in argv + assert argv[:4] == ["sudo", "-n", "-E", "bwrap"] + assert "--unshare-all" in argv + assert argv[argv.index("--uid") + 1] == str(os.getuid()) @pytest.mark.skipif( From 2cf43a16e45fb3b4f553c8f36f2a96648911229d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:10:34 -0700 Subject: [PATCH 057/128] test(sync): reproduce protected runner review gaps --- tests/test_sync_core_runner.py | 38 ++++++++++++++++++++++++++ tests/test_sync_core_supervisor.py | 38 +++++++++++++++++++++++++- tests/test_sync_determine_operation.py | 31 +++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index e727c2beb..7da0dbd78 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -6,6 +6,8 @@ from datetime import datetime, timezone from pathlib import Path, PurePosixPath +import pytest + from pdd.sync_core import ( AttestationSigner, AttestationIssue, @@ -386,6 +388,42 @@ def test_dynamic_pytest_plugins_fail_closed(tmp_path) -> None: assert "dynamic pytest_plugins" in executions[0].detail +@pytest.mark.parametrize( + "declaration", + [ + "pytest_plugins = []\npytest_plugins.append('tests.plugin')\n", + "pytest_plugins = set()\npytest_plugins.update({'tests.plugin'})\n", + "pytest_plugins = []\nglobals()['pytest_plugins'] = ['tests.plugin']\n", + "pytest_plugins = []\nsetattr(sys.modules[__name__], 'pytest_plugins', ['tests.plugin'])\n", + ], +) +def test_mutated_pytest_plugins_fail_closed(tmp_path, declaration) -> None: + root, _initial = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "tests/__init__.py").write_text("") + (root / "tests/plugin.py").write_text("VALUE = 1\n") + (root / "conftest.py").write_text("import sys\n" + declaration) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "mutated plugin declaration") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.ERROR + assert "dynamic pytest_plugins" in executions[0].detail + + +def test_unrelated_getattr_in_protected_conftest_is_allowed(tmp_path) -> None: + root, _initial = _repository(tmp_path, "def test_widget(value): assert value == 1\n") + (root / "conftest.py").write_text( + "import pytest\n" + "VALUE = getattr(type('Config', (), {'value': 1}), 'value')\n" + "@pytest.fixture\ndef value(): return VALUE\n" + ) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "ordinary reflection") + head = _git(root, "rev-parse", "HEAD") + _envelope, executions = _run(root, head, head) + assert executions[0].outcome is EvidenceOutcome.PASS + + def test_candidate_modified_importfrom_alias_helper_cannot_self_certify(tmp_path) -> None: root, _initial = _repository( tmp_path, diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 3ee4b8b0c..5a0dd26a7 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -26,6 +26,20 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( assert argv[:4] == ["sudo", "-n", "-E", "bwrap"] assert "--unshare-all" in argv assert argv[argv.index("--uid") + 1] == str(os.getuid()) + assert argv.index("--bind") < argv.index("--uid") < argv.index("--") + assert argv.index("--proc") < argv.index("--uid") + assert "--ro-bind" not in argv or argv[argv.index("--ro-bind") + 1] != "/" + + +def test_protected_runner_declares_finite_resource_limits() -> None: + from pdd.sync_core.supervisor import SupervisorLimits + + limits = SupervisorLimits() + assert 0 < limits.max_output_bytes <= 16 * 1024 * 1024 + assert 0 < limits.max_writable_bytes <= 1024 * 1024 * 1024 + assert 0 < limits.max_memory_bytes <= 4 * 1024 * 1024 * 1024 + assert 0 < limits.max_cpu_seconds <= 600 + assert 0 < limits.max_processes <= 256 @pytest.mark.skipif( @@ -70,7 +84,7 @@ def test_detached_descendant_cannot_escape_by_removing_marker(tmp_path: Path) -> "import os,subprocess,sys,time; child_env=dict(os.environ); " "child_env.pop('PDD_SUPERVISION_TOKEN', None); " "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " - "env=child_env); time.sleep(0.1)" + "env=child_env)" ) result, surviving = run_supervised( [sys.executable, "-c", parent, child, str(marker)], cwd=tmp_path, @@ -80,3 +94,25 @@ def test_detached_descendant_cannot_escape_by_removing_marker(tmp_path: Path) -> assert surviving is True time.sleep(1.2) assert not marker.exists() + + +@pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") +def test_candidate_cannot_read_absolute_host_sentinel(tmp_path: Path) -> None: + sentinel = tmp_path.parent / "protected-host-secret" + sentinel.write_text("secret", encoding="utf-8") + result, surviving = run_supervised( + [sys.executable, "-c", "import pathlib,sys; pathlib.Path(sys.argv[1]).read_text()", str(sentinel)], + cwd=tmp_path, timeout=10, env=dict(os.environ), writable_roots=(tmp_path,), + ) + assert result.returncode != 0 + assert surviving is False + + +@pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") +def test_output_is_bounded(tmp_path: Path) -> None: + result, _surviving = run_supervised( + [sys.executable, "-c", "print('x' * 20000000)"], cwd=tmp_path, + timeout=10, env=dict(os.environ), writable_roots=(tmp_path,), + ) + assert len(result.stdout.encode()) <= 16 * 1024 * 1024 + assert result.returncode != 0 diff --git a/tests/test_sync_determine_operation.py b/tests/test_sync_determine_operation.py index 5938ec4b4..bd5217732 100644 --- a/tests/test_sync_determine_operation.py +++ b/tests/test_sync_determine_operation.py @@ -4219,6 +4219,37 @@ def test_legacy_include_hash_sorts_and_deduplicates_dependencies( ).hexdigest() assert calculate_prompt_hash(prompt, hash_version=1) == expected + def test_legacy_v1_preserves_pre_versioned_include_grammar_and_missing_files( + self, pdd_test_environment + ): + prompt = pdd_test_environment / "prompts" / f"{BASENAME}_{LANGUAGE}.prompt" + body_dep = prompt.parent / "body.py" + attr_dep = prompt.parent / "attribute.py" + create_file(body_dep, "BODY = 1\n") + create_file(attr_dep, "ATTRIBUTE = 1\n") + create_file( + prompt, + "body.py\n" + "\n" + "*.py\n" + "missing.py\n", + ) + expected = hashlib.sha256(prompt.read_bytes() + body_dep.read_bytes()).hexdigest() + assert calculate_prompt_hash(prompt, hash_version=1) == expected + + def test_legacy_v1_stored_dependencies_skip_missing_and_keep_key_order( + self, pdd_test_environment + ): + prompt = pdd_test_environment / "prompts" / f"{BASENAME}_{LANGUAGE}.prompt" + first = pdd_test_environment / "a.py" + second = pdd_test_environment / "b.py" + create_file(prompt, "No includes.\n") + create_file(first, "A = 1\n") + create_file(second, "B = 1\n") + stored = {str(second): "old", str(pdd_test_environment / "missing.py"): "old", str(first): "old"} + expected = hashlib.sha256(prompt.read_bytes() + first.read_bytes() + second.read_bytes()).hexdigest() + assert calculate_prompt_hash(prompt, stored_deps=stored, hash_version=1) == expected + def test_calculate_prompt_hash_detects_dep_change_via_stored_deps(self, pdd_test_environment): """When a stored dep file changes, the composite hash must change.""" prompts_dir = pdd_test_environment / "prompts" From f314804695b4f5a424b73eee499774d77d8b34ae Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:22:41 -0700 Subject: [PATCH 058/128] fix(sync): close protected runner review gaps --- .github/workflows/unit-tests.yml | 38 +++++++-- pdd/sync_core/runner.py | 71 ++++++++++++++--- pdd/sync_core/supervisor.py | 119 +++++++++++++++++++++++------ pdd/sync_determine_operation.py | 18 ++++- tests/test_sync_core_runner.py | 33 ++++++-- tests/test_sync_core_supervisor.py | 84 ++++++++++++++++++-- 6 files changed, 309 insertions(+), 54 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 28986725f..9a705ed7d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -52,16 +52,42 @@ jobs: git config --global user.name "PDD CI" git config --global init.defaultBranch main + - name: Install dependencies + run: pip install -e ".[dev]" pytest-timeout + - name: Provision and verify protected Linux sandbox run: | sudo apt-get update sudo apt-get install --yes bubblewrap - sudo -n -E bwrap --unshare-all --die-with-parent --new-session \ - --uid "$(id -u)" --gid "$(id -g)" --ro-bind / / \ - --dev /dev --proc /proc -- /bin/true - - - name: Install dependencies - run: pip install -e ".[dev]" pytest-timeout + private_root="$(mktemp -d)" + chmod 700 "$private_root" + mkdir -m 700 "$private_root/scratch" + PRIVATE_ROOT="$private_root" python - <<'PY' + import os + import sys + from pathlib import Path + from pdd.sync_core.supervisor import run_supervised + + root = Path(os.environ["PRIVATE_ROOT"]) + script = """from pathlib import Path +Path('ok').write_text('ok') +try: + Path('../outside').write_text('forbidden') +except OSError: + pass +else: + raise SystemExit('write escaped private scratch root') +""" + result, surviving = run_supervised( + [sys.executable, "-c", script], + cwd=root / "scratch", timeout=10, env=dict(os.environ), + writable_roots=(root / "scratch",), + ) + assert result.returncode == 0, result.stderr + assert not surviving + assert (root / "scratch" / "ok").read_text() == "ok" + assert not (root / "outside").exists() + PY - name: Validate architecture vs prompt includes (fixture smoke) run: pdd checkup --validate-arch-includes --project-root tests/fixtures/arch_include_validate_ok diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index e0f7ed089..8a11dd4cd 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -1,5 +1,6 @@ """Trusted validator adapters and pass-only normalized evidence outcomes.""" # pylint: disable=too-many-lines,too-many-boolean-expressions,too-many-locals +# pylint: disable=too-many-arguments from __future__ import annotations @@ -122,7 +123,7 @@ def _local_module_paths( *, code_under_test_paths: frozenset[PurePosixPath] = frozenset(), ) -> tuple[set[PurePosixPath], bool]: - # pylint: disable=too-many-locals,too-many-branches + # pylint: disable=too-many-locals,too-many-branches,too-many-statements """Resolve repository-local Python imports without executing candidate code.""" try: tree = ast.parse(source) @@ -132,6 +133,18 @@ def _local_module_paths( dynamic_pytest_plugins = False importlib_names = {"importlib"} loader_names = {"__import__"} + plugin_aliases = {"pytest_plugins"} + for candidate in ast.walk(tree): + if ( + isinstance(candidate, (ast.Assign, ast.AnnAssign)) + and isinstance(candidate.value, ast.Name) + and candidate.value.id in plugin_aliases + ): + plugin_aliases.update( + target.id for target in _pytest_plugin_declaration_targets(candidate) + if isinstance(target, ast.Name) + ) + pytest_plugins_declared = False for node in ast.walk(tree): if isinstance(node, ast.Import): modules.update(alias.name for alias in node.names) @@ -153,6 +166,7 @@ def _local_module_paths( if node.module == "importlib" and alias.name == "import_module": loader_names.add(alias.asname or alias.name) elif _declares_pytest_plugins(_pytest_plugin_declaration_targets(node)): + pytest_plugins_declared = True declared, dynamic = _pytest_plugin_modules(node.value) modules.update(declared) dynamic_pytest_plugins = dynamic_pytest_plugins or dynamic @@ -166,12 +180,47 @@ def _local_module_paths( dynamic_pytest_plugins = True elif isinstance(node, ast.Call) and ( isinstance(node.func, ast.Name) - and node.func.id in {"getattr", "exec", "eval", "compile", "run_path", "run_module"} + and node.func.id in {"exec", "eval", "compile", "run_path", "run_module"} or isinstance(node.func, ast.Attribute) and node.func.attr in { "spec_from_file_location", "load_module", "exec_module", "run_path", "run_module", } + or isinstance(node.func, ast.Name) and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[0], ast.Name) + and node.args[0].id in importlib_names + and isinstance(node.args[1], ast.Constant) + and node.args[1].value == "import_module" + ): + dynamic_pytest_plugins = True + elif isinstance(node, ast.Call) and ( + pytest_plugins_declared + or isinstance(node.func, ast.Name) and node.func.id == "setattr" + ): + if ( + isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id in plugin_aliases + ) or ( + isinstance(node.func, ast.Name) + and node.func.id in {"globals", "locals", "exec", "eval"} + ) or ( + isinstance(node.func, ast.Name) and node.func.id == "setattr" + and any( + isinstance(arg, ast.Constant) and arg.value == "pytest_plugins" + for arg in node.args + ) + ): + dynamic_pytest_plugins = True + elif isinstance(node, (ast.Assign, ast.AnnAssign)) and any( + isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Call) + and isinstance(target.value.func, ast.Name) + and target.value.func.id in {"globals", "locals"} + and isinstance(target.slice, ast.Constant) + and target.slice.value == "pytest_plugins" + for target in _pytest_plugin_declaration_targets(node) ): dynamic_pytest_plugins = True elif isinstance(node, (ast.Assign, ast.AnnAssign)): @@ -418,11 +467,11 @@ def _config_loads_plugin(root: Path, ref: str) -> bool: def _managed_subprocess( command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], - writable_roots: tuple[Path, ...], + writable_roots: tuple[Path, ...], writable_files: tuple[Path, ...] = (), ) -> tuple[subprocess.CompletedProcess[str], bool]: """Run an untrusted command in a networkless sandbox and reap its group.""" return run_supervised(command, cwd=cwd, timeout=timeout, env=env, - writable_roots=writable_roots) + writable_roots=writable_roots, writable_files=writable_files) def runner_identity_digest( @@ -637,13 +686,14 @@ def _run_test_node( ).hexdigest() with tempfile.TemporaryDirectory(prefix="pdd-trusted-runner-") as directory: temporary = Path(directory) - home = temporary / "home" - home.mkdir(mode=0o700) + home = temporary / "scratch" / "home" + home.mkdir(mode=0o700, parents=True) junit = temporary / "junit.xml" + junit.touch(mode=0o600) result, surviving = _managed_subprocess( [*command, f"--junitxml={junit}"], cwd=root, timeout=timeout_seconds, env=_pytest_environment(home), - writable_roots=(temporary,), + writable_roots=(home.parent,), writable_files=(junit,), ) if result.returncode == 124: return RunnerExecution( @@ -675,9 +725,10 @@ def _collect_node_ids( """Collect exact pytest node IDs through the protected adapter.""" with tempfile.TemporaryDirectory(prefix="pdd-trusted-collection-") as directory: temporary = Path(directory) - home = temporary / "home" - home.mkdir(mode=0o700) + home = temporary / "scratch" / "home" + home.mkdir(mode=0o700, parents=True) collection_output = temporary / "node-ids.json" + collection_output.touch(mode=0o600) pytest_args = [ "--collect-only", "-q", @@ -705,7 +756,7 @@ def _collect_node_ids( command, cwd=temporary, timeout=timeout_seconds, env=_pytest_environment(home) | { "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), - }, writable_roots=(temporary,), + }, writable_roots=(home.parent,), writable_files=(collection_output,), ) if result.returncode == 124: return ( diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 04674de1e..227587e8c 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -1,4 +1,5 @@ """Fail-closed OS sandbox and complete process-group supervision.""" +# pylint: disable=too-many-arguments from __future__ import annotations @@ -11,9 +12,49 @@ import threading import time import uuid +from dataclasses import dataclass from pathlib import Path +@dataclass(frozen=True) +class SupervisorLimits: + """Hard limits applied to every untrusted validator process tree.""" + + max_output_bytes: int = 16 * 1024 * 1024 + max_writable_bytes: int = 512 * 1024 * 1024 + max_memory_bytes: int = 2 * 1024 * 1024 * 1024 + max_cpu_seconds: int = 300 + max_processes: int = 128 + + +def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: + """Return the minimal host trees needed to start the configured interpreter.""" + roots = {cwd.resolve(), Path(sys.prefix).resolve()} + executable = shutil.which(command[0]) or command[0] + roots.add(Path(executable).resolve().parent) + for candidate in ("/bin", "/usr", "/lib", "/lib64"): + path = Path(candidate) + if path.exists(): + roots.add(path.resolve()) + return tuple(sorted(roots, key=lambda item: (len(item.parts), str(item)))) + + +def _limited_command(command: list[str], limits: SupervisorLimits) -> list[str]: + """Apply non-raiseable POSIX limits after the namespace uid drop.""" + script = ( + "import os,resource,sys;" + "v=[int(x) for x in sys.argv[1:6]];" + "resource.setrlimit(resource.RLIMIT_AS,(v[0],v[0]));" + "resource.setrlimit(resource.RLIMIT_CPU,(v[1],v[1]));" + "resource.setrlimit(resource.RLIMIT_NPROC,(v[2],v[2]));" + "resource.setrlimit(resource.RLIMIT_FSIZE,(v[3],v[3]));" + "resource.setrlimit(resource.RLIMIT_NOFILE,(v[4],v[4]));" + "os.execvpe(sys.argv[6],sys.argv[6:],os.environ)" + ) + return [sys.executable, "-c", script, str(limits.max_memory_bytes), + str(limits.max_cpu_seconds), str(limits.max_processes), + str(limits.max_writable_bytes), "256", *command] + def _supervised_descendants(token: str) -> set[int]: """Find descendants carrying the unforgeable per-run environment marker.""" found: set[int] = set() @@ -79,48 +120,56 @@ def _live_processes(pids: set[int]) -> set[int]: def _sandbox_command( - command: list[str], writable_roots: tuple[Path, ...] + command: list[str], writable_roots: tuple[Path, ...], *, cwd: Path | None = None, + writable_files: tuple[Path, ...] = (), limits: SupervisorLimits = SupervisorLimits(), ) -> tuple[list[str], Path | None]: """Return an explicitly detected macOS/Linux sandbox command.""" - if sys.platform == "darwin" and shutil.which("sandbox-exec"): - rules = ["(version 1)", "(allow default)", "(deny network*)", "(deny file-write*)", - '(allow file-write* (literal "/dev/null"))'] - for item in writable_roots: - rules.append(f'(allow file-write* (subpath "{item.resolve()}"))') - descriptor, name = tempfile.mkstemp(prefix="pdd-sandbox-", suffix=".sb") - os.close(descriptor) - profile = Path(name) - profile.write_text("\n".join(rules), encoding="utf-8") - return ["sandbox-exec", "-f", str(profile), *command], profile + if sys.platform == "darwin": + raise RuntimeError( + "unsupported protected sandbox: macOS cannot prove process lifetime isolation" + ) if sys.platform.startswith("linux") and shutil.which("bwrap"): elevated = bool(shutil.which("sudo")) and subprocess.run( ["sudo", "-n", "true"], capture_output=True, check=False, ).returncode == 0 prefix = ["sudo", "-n", "-E"] if elevated else [] + workdir = (cwd or Path.cwd()).resolve() argv = [*prefix, "bwrap", "--unshare-all", "--die-with-parent", - "--new-session", "--uid", str(os.getuid()), "--gid", - str(os.getgid()), "--ro-bind", "/", "/", - "--dev", "/dev", "--proc", "/proc"] + "--new-session", "--tmpfs", "/", "--dir", "/tmp"] + for item in _runtime_roots(command, workdir): + argv.extend(("--ro-bind", str(item), str(item))) + argv.extend(("--dev", "/dev", "--proc", "/proc")) for item in writable_roots: resolved = str(item.resolve()) argv.extend(("--bind", resolved, resolved)) - return [*argv, "--", *command], None + for item in writable_files: + resolved = str(item.resolve()) + argv.extend(("--bind", resolved, resolved)) + argv.extend(("--uid", str(os.getuid()), "--gid", str(os.getgid()), + "--chdir", str(workdir))) + return [*argv, "--", *_limited_command(command, limits)], None raise RuntimeError("unsupported sandbox platform or mechanism") def run_supervised( command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], - writable_roots: tuple[Path, ...], + writable_roots: tuple[Path, ...], writable_files: tuple[Path, ...] = (), + limits: SupervisorLimits = SupervisorLimits(), ) -> tuple[subprocess.CompletedProcess[str], bool]: """Run sandboxed and terminate marked descendants across session changes.""" # pylint: disable=consider-using-with,too-many-locals,too-many-branches,too-many-statements try: - argv, profile = _sandbox_command(command, writable_roots) + argv, profile = _sandbox_command( + command, writable_roots, cwd=cwd, writable_files=writable_files, + limits=limits, + ) except RuntimeError as exc: return subprocess.CompletedProcess(command, 125, "", str(exc)), False token = uuid.uuid4().hex + stdout_file = tempfile.TemporaryFile(mode="w+", encoding="utf-8") + stderr_file = tempfile.TemporaryFile(mode="w+", encoding="utf-8") process = subprocess.Popen( - argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + argv, cwd=cwd, stdout=stdout_file, stderr=stderr_file, text=True, env=env | {"PYTHONDONTWRITEBYTECODE": "1", "PDD_SUPERVISION_TOKEN": token, "TMPDIR": str(writable_roots[0].resolve()), @@ -140,13 +189,23 @@ def track_process_tree() -> None: tracker = threading.Thread(target=track_process_tree, daemon=True) tracker.start() deadline = time.monotonic() + timeout + output_limited = False while True: try: - stdout, stderr = process.communicate( + process.communicate( timeout=max(0.01, min(0.1, deadline - time.monotonic())) ) break except subprocess.TimeoutExpired: + writable_size = sum( + path.stat().st_size for root in writable_roots + for path in root.rglob("*") if path.is_file() + ) + if writable_size > limits.max_writable_bytes: + output_limited = True + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + break descendants = _supervised_descendants(token) - {process.pid} if process.poll() is not None and descendants: surviving = True @@ -166,7 +225,7 @@ def track_process_tree() -> None: os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass - stdout, stderr = process.communicate() + process.communicate() break tracking_done.set() tracker.join(timeout=1) @@ -189,5 +248,21 @@ def track_process_tree() -> None: pass if profile is not None: profile.unlink(missing_ok=True) - return subprocess.CompletedProcess(command, 124 if timed_out else process.returncode, - stdout, stderr), surviving + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(limits.max_output_bytes + 1) + stderr = stderr_file.read(limits.max_output_bytes + 1) + stdout_file.close() + stderr_file.close() + encoded = stdout.encode() + if len(encoded) > limits.max_output_bytes: + stdout = encoded[:limits.max_output_bytes].decode("utf-8", errors="replace") + output_limited = True + encoded = stderr.encode() + if len(encoded) > limits.max_output_bytes: + stderr = encoded[:limits.max_output_bytes].decode("utf-8", errors="replace") + output_limited = True + return subprocess.CompletedProcess( + command, 125 if output_limited else (124 if timed_out else process.returncode), + stdout, stderr, + ), surviving diff --git a/pdd/sync_determine_operation.py b/pdd/sync_determine_operation.py index baaf10eb1..5368b7e0a 100644 --- a/pdd/sync_determine_operation.py +++ b/pdd/sync_determine_operation.py @@ -7,6 +7,7 @@ """ import os +import re import sys import glob import json @@ -1791,6 +1792,13 @@ def _legacy_dependency_path(prompt_path: Path, declared: str) -> Optional[Path]: return None +def _legacy_include_references(content: str) -> list[str]: + """Freeze the exact pre-versioned include grammar for v1 fingerprints.""" + xml = re.findall(r"]*>(.*?)", content) + backtick = re.findall(r"```<([^>]*?)>```", content) + return xml + backtick + + def calculate_prompt_hash( prompt_path: Path, stored_deps: Optional[Dict[str, str]] = None, @@ -1820,9 +1828,13 @@ def calculate_prompt_hash( return None from pdd.sync_core.includes import parse_include_references - references = parse_include_references(prompt_content) + references = ( + _legacy_include_references(prompt_content) + if hash_version == 1 else + [reference.path for reference in parse_include_references(prompt_content)] + ) declared_dependencies = ( - [reference.path for reference in references] + references if references else list((stored_deps or {}).keys()) ) if hash_version == 1: @@ -1831,6 +1843,8 @@ def calculate_prompt_hash( for declared in declared_dependencies: candidate = _legacy_dependency_path(prompt_path.resolve(), declared) if candidate is None: + if hash_version == 1: + continue return None resolved_dependencies.append(candidate.resolve()) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 7da0dbd78..9041e9f02 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -19,7 +19,11 @@ VerificationProfile, run_profile, ) -from pdd.sync_core.runner import pytest_validator_config_digest +from pdd.sync_core.runner import ( + _has_dynamic_pytest_plugins, + _local_module_paths, + pytest_validator_config_digest, +) UNIT = UnitId("repository-1", PurePosixPath("prompts/widget_python.prompt"), "python") @@ -68,6 +72,8 @@ def _run( head: str, code_under_test_paths: tuple[PurePosixPath, ...] = (), ): + if sys.platform == "darwin": + pytest.skip("protected validators fail closed without Linux namespaces") return run_profile( root, _profile(root, base, code_under_test_paths), @@ -395,6 +401,8 @@ def test_dynamic_pytest_plugins_fail_closed(tmp_path) -> None: "pytest_plugins = set()\npytest_plugins.update({'tests.plugin'})\n", "pytest_plugins = []\nglobals()['pytest_plugins'] = ['tests.plugin']\n", "pytest_plugins = []\nsetattr(sys.modules[__name__], 'pytest_plugins', ['tests.plugin'])\n", + "pytest_plugins = []\nalias = pytest_plugins\nalias.append('tests.plugin')\n", + "setattr(sys.modules[__name__], 'pytest_plugins', ['tests.plugin'])\n", ], ) def test_mutated_pytest_plugins_fail_closed(tmp_path, declaration) -> None: @@ -405,9 +413,9 @@ def test_mutated_pytest_plugins_fail_closed(tmp_path, declaration) -> None: _git(root, "add", ".") _git(root, "commit", "-q", "-m", "mutated plugin declaration") head = _git(root, "rev-parse", "HEAD") - _envelope, executions = _run(root, head, head) - assert executions[0].outcome is EvidenceOutcome.ERROR - assert "dynamic pytest_plugins" in executions[0].detail + assert _has_dynamic_pytest_plugins( + root, head, (PurePosixPath("tests/test_widget.py"),) + ) def test_unrelated_getattr_in_protected_conftest_is_allowed(tmp_path) -> None: @@ -420,8 +428,21 @@ def test_unrelated_getattr_in_protected_conftest_is_allowed(tmp_path) -> None: _git(root, "add", ".") _git(root, "commit", "-q", "-m", "ordinary reflection") head = _git(root, "rev-parse", "HEAD") - _envelope, executions = _run(root, head, head) - assert executions[0].outcome is EvidenceOutcome.PASS + assert not _has_dynamic_pytest_plugins( + root, head, (PurePosixPath("tests/test_widget.py"),) + ) + + +def test_repository_conftest_ordinary_getattr_passes_plugin_preflight( + monkeypatch, +) -> None: + root = Path(__file__).resolve().parents[1] + source = (root / "tests/conftest.py").read_bytes() + monkeypatch.setattr("pdd.sync_core.runner.read_git_blob", lambda *_args: None) + _paths, dynamic = _local_module_paths( + root, "HEAD", PurePosixPath("tests/conftest.py"), source + ) + assert not dynamic def test_candidate_modified_importfrom_alias_helper_cannot_self_certify(tmp_path) -> None: diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 5a0dd26a7..e1c2b0855 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -9,7 +9,7 @@ import pytest -from pdd.sync_core.supervisor import _sandbox_command, run_supervised +from pdd.sync_core.supervisor import SupervisorLimits, _sandbox_command, run_supervised def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( @@ -32,8 +32,6 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( def test_protected_runner_declares_finite_resource_limits() -> None: - from pdd.sync_core.supervisor import SupervisorLimits - limits = SupervisorLimits() assert 0 < limits.max_output_bytes <= 16 * 1024 * 1024 assert 0 < limits.max_writable_bytes <= 1024 * 1024 * 1024 @@ -43,8 +41,8 @@ def test_protected_runner_declares_finite_resource_limits() -> None: @pytest.mark.skipif( - not (shutil.which("sandbox-exec") or shutil.which("bwrap")), - reason="requires a real supported sandbox", + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux kernel namespace containment", ) def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: marker = tmp_path / "escaped" @@ -71,8 +69,8 @@ def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: @pytest.mark.skipif( - not (shutil.which("sandbox-exec") or shutil.which("bwrap")), - reason="requires a real supported sandbox", + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux kernel namespace containment", ) def test_detached_descendant_cannot_escape_by_removing_marker(tmp_path: Path) -> None: marker = tmp_path / "escaped-without-marker" @@ -101,13 +99,44 @@ def test_candidate_cannot_read_absolute_host_sentinel(tmp_path: Path) -> None: sentinel = tmp_path.parent / "protected-host-secret" sentinel.write_text("secret", encoding="utf-8") result, surviving = run_supervised( - [sys.executable, "-c", "import pathlib,sys; pathlib.Path(sys.argv[1]).read_text()", str(sentinel)], + [ + sys.executable, "-c", + "import pathlib,sys; pathlib.Path(sys.argv[1]).read_text()", + str(sentinel), + ], cwd=tmp_path, timeout=10, env=dict(os.environ), writable_roots=(tmp_path,), ) assert result.returncode != 0 assert surviving is False +@pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") +def test_immediate_detached_child_cannot_forge_checker_result_channel( + tmp_path: Path, +) -> None: + scratch = tmp_path / "scratch" + scratch.mkdir() + result_channel = tmp_path / "junit.xml" + result_channel.write_text("checker-owned", encoding="utf-8") + child = ( + "import os,sys,time; os.setsid(); time.sleep(.1); " + "open(sys.argv[1], 'w').write('forged')" + ) + parent = ( + "import os,subprocess,sys; env=dict(os.environ); " + "env.pop('PDD_SUPERVISION_TOKEN', None); " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], env=env)" + ) + completed, _surviving = run_supervised( + [sys.executable, "-c", parent, child, str(result_channel)], + cwd=scratch, timeout=10, env=dict(os.environ), + writable_roots=(scratch,), writable_files=(result_channel,), + ) + assert completed.returncode == 0 + time.sleep(0.3) + assert result_channel.read_text(encoding="utf-8") == "checker-owned" + + @pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") def test_output_is_bounded(tmp_path: Path) -> None: result, _surviving = run_supervised( @@ -116,3 +145,42 @@ def test_output_is_bounded(tmp_path: Path) -> None: ) assert len(result.stdout.encode()) <= 16 * 1024 * 1024 assert result.returncode != 0 + + +@pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") +@pytest.mark.parametrize( + "program", + [ + "bytearray(256 * 1024 * 1024)", + "open('large', 'wb').truncate(8 * 1024 * 1024)", + "while True: pass", + "import subprocess,sys,time; " + "children=[subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(2)']) " + "for _ in range(16)]; [child.wait() for child in children]", + ], +) +def test_memory_disk_and_process_limits_fail_closed(tmp_path: Path, program: str) -> None: + limits = SupervisorLimits( + max_memory_bytes=128 * 1024 * 1024, + max_writable_bytes=1024 * 1024, + max_cpu_seconds=1, + max_processes=4, + ) + result, _surviving = run_supervised( + [sys.executable, "-c", program], cwd=tmp_path, timeout=10, + env=dict(os.environ), writable_roots=(tmp_path,), limits=limits, + ) + assert result.returncode != 0 + + +def test_macos_fails_closed_without_kernel_lifetime_containment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(sys, "platform", "darwin") + result, surviving = run_supervised( + ["/bin/true"], cwd=tmp_path, timeout=1, env={}, + writable_roots=(tmp_path,), + ) + assert result.returncode == 125 + assert "cannot prove process lifetime isolation" in result.stderr + assert surviving is False From ad5487d2d1e6100bc7911a2a4f34550d7d0409bd Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:24:36 -0700 Subject: [PATCH 059/128] fix(sync): repair Linux smoke workflow syntax --- .github/workflows/unit-tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9a705ed7d..c158ded21 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -70,14 +70,14 @@ jobs: root = Path(os.environ["PRIVATE_ROOT"]) script = """from pathlib import Path -Path('ok').write_text('ok') -try: - Path('../outside').write_text('forbidden') -except OSError: - pass -else: - raise SystemExit('write escaped private scratch root') -""" + Path('ok').write_text('ok') + try: + Path('../outside').write_text('forbidden') + except OSError: + pass + else: + raise SystemExit('write escaped private scratch root') + """ result, surviving = run_supervised( [sys.executable, "-c", script], cwd=root / "scratch", timeout=10, env=dict(os.environ), From d7955acab45b18f4481d1b11787555c2a4c116b7 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:28:13 -0700 Subject: [PATCH 060/128] fix(sync): drop sandbox privileges after mounts --- pdd/sync_core/supervisor.py | 12 +++++++++--- tests/test_sync_core_supervisor.py | 7 ++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 227587e8c..75b81e002 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -133,6 +133,9 @@ def _sandbox_command( ["sudo", "-n", "true"], capture_output=True, check=False, ).returncode == 0 prefix = ["sudo", "-n", "-E"] if elevated else [] + setpriv = shutil.which("setpriv") if elevated else None + if elevated and setpriv is None: + raise RuntimeError("protected sandbox requires setpriv for post-mount uid drop") workdir = (cwd or Path.cwd()).resolve() argv = [*prefix, "bwrap", "--unshare-all", "--die-with-parent", "--new-session", "--tmpfs", "/", "--dir", "/tmp"] @@ -145,9 +148,12 @@ def _sandbox_command( for item in writable_files: resolved = str(item.resolve()) argv.extend(("--bind", resolved, resolved)) - argv.extend(("--uid", str(os.getuid()), "--gid", str(os.getgid()), - "--chdir", str(workdir))) - return [*argv, "--", *_limited_command(command, limits)], None + argv.extend(("--chdir", str(workdir))) + drop = ( + [setpriv, "--reuid", str(os.getuid()), "--regid", str(os.getgid()), + "--clear-groups", "--"] if setpriv else [] + ) + return [*argv, "--", *drop, *_limited_command(command, limits)], None raise RuntimeError("unsupported sandbox platform or mechanism") diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index e1c2b0855..a986fbda6 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -25,9 +25,10 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( assert profile is None assert argv[:4] == ["sudo", "-n", "-E", "bwrap"] assert "--unshare-all" in argv - assert argv[argv.index("--uid") + 1] == str(os.getuid()) - assert argv.index("--bind") < argv.index("--uid") < argv.index("--") - assert argv.index("--proc") < argv.index("--uid") + separator = argv.index("--") + assert argv.index("--bind") < separator < argv.index("--reuid") + assert argv[argv.index("--reuid") + 1] == str(os.getuid()) + assert argv.index("--proc") < separator assert "--ro-bind" not in argv or argv[argv.index("--ro-bind") + 1] != "/" From bb3e1a88ac3f568317ae4198b285ff07614fb5e3 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:31:48 -0700 Subject: [PATCH 061/128] fix(sync): preopen private sandbox mounts --- pdd/sync_core/supervisor.py | 39 ++++++++++++++++++++++-------- tests/test_sync_core_supervisor.py | 16 ++++++------ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 75b81e002..9a199aa32 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -4,6 +4,7 @@ from __future__ import annotations import os +import json import shutil import signal import subprocess @@ -55,6 +56,22 @@ def _limited_command(command: list[str], limits: SupervisorLimits) -> list[str]: str(limits.max_cpu_seconds), str(limits.max_processes), str(limits.max_writable_bytes), "256", *command] + +def _fd_bound_bwrap(argv: list[str], sources: list[Path], elevated: bool) -> list[str]: + """Open bind sources before replacing the namespace root.""" + helper = ( + "import json,os,sys;" + "argv=json.loads(sys.argv[1]);paths=json.loads(sys.argv[2]);" + "fds=[os.open(path,os.O_PATH) for path in paths];" + "[os.set_inheritable(fd,True) for fd in fds];" + "argv=[('/proc/self/fd/'+str(fds[int(x[4:-1])])) " + "if x.startswith('@FD:') else x for x in argv];" + "os.execvp(argv[0],argv)" + ) + prefix = ["sudo", "-n", "-E"] if elevated else [] + return [*prefix, sys.executable, "-c", helper, + json.dumps(argv), json.dumps([str(path) for path in sources])] + def _supervised_descendants(token: str) -> set[int]: """Find descendants carrying the unforgeable per-run environment marker.""" found: set[int] = set() @@ -132,28 +149,30 @@ def _sandbox_command( elevated = bool(shutil.which("sudo")) and subprocess.run( ["sudo", "-n", "true"], capture_output=True, check=False, ).returncode == 0 - prefix = ["sudo", "-n", "-E"] if elevated else [] setpriv = shutil.which("setpriv") if elevated else None if elevated and setpriv is None: raise RuntimeError("protected sandbox requires setpriv for post-mount uid drop") workdir = (cwd or Path.cwd()).resolve() - argv = [*prefix, "bwrap", "--unshare-all", "--die-with-parent", - "--new-session", "--tmpfs", "/", "--dir", "/tmp"] + argv = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session", + "--tmpfs", "/", "--proc", "/proc", "--dir", "/tmp"] + sources: list[Path] = [] + def bind(option: str, source: Path) -> None: + sources.append(source) + argv.extend((option, f"@FD:{len(sources) - 1}@", str(source))) for item in _runtime_roots(command, workdir): - argv.extend(("--ro-bind", str(item), str(item))) - argv.extend(("--dev", "/dev", "--proc", "/proc")) + bind("--ro-bind", item) + argv.extend(("--dev", "/dev")) for item in writable_roots: - resolved = str(item.resolve()) - argv.extend(("--bind", resolved, resolved)) + bind("--bind", item.resolve()) for item in writable_files: - resolved = str(item.resolve()) - argv.extend(("--bind", resolved, resolved)) + bind("--bind", item.resolve()) argv.extend(("--chdir", str(workdir))) drop = ( [setpriv, "--reuid", str(os.getuid()), "--regid", str(os.getgid()), "--clear-groups", "--"] if setpriv else [] ) - return [*argv, "--", *drop, *_limited_command(command, limits)], None + argv.extend(("--", *drop, *_limited_command(command, limits))) + return _fd_bound_bwrap(argv, sources, elevated), None raise RuntimeError("unsupported sandbox platform or mechanism") diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index a986fbda6..8ae26e5f1 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -1,6 +1,7 @@ """Adversarial tests for complete protected subprocess supervision.""" import os +import json import shutil import subprocess import sys @@ -23,13 +24,14 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( ) argv, profile = _sandbox_command(["/bin/true"], (tmp_path,)) assert profile is None - assert argv[:4] == ["sudo", "-n", "-E", "bwrap"] - assert "--unshare-all" in argv - separator = argv.index("--") - assert argv.index("--bind") < separator < argv.index("--reuid") - assert argv[argv.index("--reuid") + 1] == str(os.getuid()) - assert argv.index("--proc") < separator - assert "--ro-bind" not in argv or argv[argv.index("--ro-bind") + 1] != "/" + assert argv[:3] == ["sudo", "-n", "-E"] + bwrap = json.loads(argv[-2]) + assert "--unshare-all" in bwrap + separator = bwrap.index("--") + assert bwrap.index("--bind") < separator < bwrap.index("--reuid") + assert bwrap[bwrap.index("--reuid") + 1] == str(os.getuid()) + assert bwrap.index("--proc") < separator + assert bwrap[bwrap.index("--ro-bind") + 1].startswith("@FD:") def test_protected_runner_declares_finite_resource_limits() -> None: From 6f7e70af253ba3944f58e75ad7d15c42429b2f92 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:35:34 -0700 Subject: [PATCH 062/128] fix(sync): stage private sandbox bind sources --- pdd/sync_core/supervisor.py | 39 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 9a199aa32..49c13e91a 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -57,19 +57,28 @@ def _limited_command(command: list[str], limits: SupervisorLimits) -> list[str]: str(limits.max_writable_bytes), "256", *command] -def _fd_bound_bwrap(argv: list[str], sources: list[Path], elevated: bool) -> list[str]: - """Open bind sources before replacing the namespace root.""" - helper = ( - "import json,os,sys;" - "argv=json.loads(sys.argv[1]);paths=json.loads(sys.argv[2]);" - "fds=[os.open(path,os.O_PATH) for path in paths];" - "[os.set_inheritable(fd,True) for fd in fds];" - "argv=[('/proc/self/fd/'+str(fds[int(x[4:-1])])) " - "if x.startswith('@FD:') else x for x in argv];" - "os.execvp(argv[0],argv)" - ) - prefix = ["sudo", "-n", "-E"] if elevated else [] - return [*prefix, sys.executable, "-c", helper, +def _staged_bwrap(argv: list[str], sources: list[Path]) -> list[str]: + """Stage exact bind mounts before replacing the namespace root.""" + helper = "\n".join(( + "import json,os,pathlib,shutil,subprocess,sys,tempfile", + "argv=json.loads(sys.argv[1]); paths=json.loads(sys.argv[2])", + "base=pathlib.Path(tempfile.mkdtemp(prefix='pdd-binds-',dir='/run'))", + "os.chmod(base,0o755); staged=[]", + "try:", + " for index,source in enumerate(paths):", + " source=pathlib.Path(source); target=base/str(index)", + " target.mkdir() if source.is_dir() else target.touch()", + " subprocess.run(['mount','--bind',str(source),str(target)],check=True)", + " staged.append(target)", + " argv=[str(staged[int(x[4:-1])]) if x.startswith('@FD:') else x for x in argv]", + " result=subprocess.run(argv,check=False)", + "finally:", + " for target in reversed(staged):", + " subprocess.run(['umount',str(target)],check=False)", + " shutil.rmtree(base,ignore_errors=True)", + "raise SystemExit(result.returncode)", + )) + return ["sudo", "-n", "-E", sys.executable, "-c", helper, json.dumps(argv), json.dumps([str(path) for path in sources])] def _supervised_descendants(token: str) -> set[int]: @@ -149,6 +158,8 @@ def _sandbox_command( elevated = bool(shutil.which("sudo")) and subprocess.run( ["sudo", "-n", "true"], capture_output=True, check=False, ).returncode == 0 + if not elevated: + raise RuntimeError("protected sandbox requires privileged bind staging") setpriv = shutil.which("setpriv") if elevated else None if elevated and setpriv is None: raise RuntimeError("protected sandbox requires setpriv for post-mount uid drop") @@ -172,7 +183,7 @@ def bind(option: str, source: Path) -> None: "--clear-groups", "--"] if setpriv else [] ) argv.extend(("--", *drop, *_limited_command(command, limits))) - return _fd_bound_bwrap(argv, sources, elevated), None + return _staged_bwrap(argv, sources), None raise RuntimeError("unsupported sandbox platform or mechanism") From 6d9f1aad6697ee8a13e5badd56b0e03153ac42e5 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:38:34 -0700 Subject: [PATCH 063/128] fix(sync): create isolated mount ancestry --- pdd/sync_core/supervisor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 49c13e91a..5fb6b5dc7 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -167,7 +167,16 @@ def _sandbox_command( argv = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session", "--tmpfs", "/", "--proc", "/proc", "--dir", "/tmp"] sources: list[Path] = [] + destination_dirs = {Path("/tmp")} def bind(option: str, source: Path) -> None: + missing = [] + parent = source.parent + while parent != Path("/") and parent not in destination_dirs: + missing.append(parent) + parent = parent.parent + for directory in reversed(missing): + argv.extend(("--dir", str(directory))) + destination_dirs.add(directory) sources.append(source) argv.extend((option, f"@FD:{len(sources) - 1}@", str(source))) for item in _runtime_roots(command, workdir): From 7e36c74005f4f608e5d51074250a0ffb693c0c25 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:41:31 -0700 Subject: [PATCH 064/128] fix(sync): retain host uid in protected namespace --- pdd/sync_core/supervisor.py | 3 ++- tests/test_sync_core_supervisor.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 5fb6b5dc7..c3ae83ade 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -164,7 +164,8 @@ def _sandbox_command( if elevated and setpriv is None: raise RuntimeError("protected sandbox requires setpriv for post-mount uid drop") workdir = (cwd or Path.cwd()).resolve() - argv = ["bwrap", "--unshare-all", "--die-with-parent", "--new-session", + argv = ["bwrap", "--unshare-ipc", "--unshare-pid", "--unshare-net", + "--unshare-uts", "--unshare-cgroup", "--die-with-parent", "--new-session", "--tmpfs", "/", "--proc", "/proc", "--dir", "/tmp"] sources: list[Path] = [] destination_dirs = {Path("/tmp")} diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 8ae26e5f1..3484ef45c 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -26,7 +26,8 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( assert profile is None assert argv[:3] == ["sudo", "-n", "-E"] bwrap = json.loads(argv[-2]) - assert "--unshare-all" in bwrap + assert {"--unshare-pid", "--unshare-net", "--unshare-cgroup"} <= set(bwrap) + assert "--unshare-user" not in bwrap separator = bwrap.index("--") assert bwrap.index("--bind") < separator < bwrap.index("--reuid") assert bwrap[bwrap.index("--reuid") + 1] == str(os.getuid()) From afa7322779eb67b8c610b9d286d12de529019ac5 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 14:44:11 -0700 Subject: [PATCH 065/128] fix(sync): preserve runtime loader mount aliases --- pdd/sync_core/supervisor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index c3ae83ade..393b6b026 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -30,13 +30,13 @@ class SupervisorLimits: def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: """Return the minimal host trees needed to start the configured interpreter.""" - roots = {cwd.resolve(), Path(sys.prefix).resolve()} + roots = {cwd.resolve(), Path(sys.prefix)} executable = shutil.which(command[0]) or command[0] - roots.add(Path(executable).resolve().parent) + roots.add(Path(executable).parent) for candidate in ("/bin", "/usr", "/lib", "/lib64"): path = Path(candidate) if path.exists(): - roots.add(path.resolve()) + roots.add(path) return tuple(sorted(roots, key=lambda item: (len(item.parts), str(item)))) From b003364489c34fccca698bf42572cde55d9d2664 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 15:08:05 -0700 Subject: [PATCH 066/128] fix(sync): mount exact protected repository inputs --- pdd/sync_core/runner.py | 6 +++++- pdd/sync_core/supervisor.py | 6 +++++- tests/test_sync_core_supervisor.py | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 8a11dd4cd..ada6d90db 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -468,10 +468,12 @@ def _config_loads_plugin(root: Path, ref: str) -> bool: def _managed_subprocess( command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], writable_roots: tuple[Path, ...], writable_files: tuple[Path, ...] = (), + readable_roots: tuple[Path, ...] = (), ) -> tuple[subprocess.CompletedProcess[str], bool]: """Run an untrusted command in a networkless sandbox and reap its group.""" return run_supervised(command, cwd=cwd, timeout=timeout, env=env, - writable_roots=writable_roots, writable_files=writable_files) + writable_roots=writable_roots, writable_files=writable_files, + readable_roots=readable_roots) def runner_identity_digest( @@ -694,6 +696,7 @@ def _run_test_node( [*command, f"--junitxml={junit}"], cwd=root, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), writable_files=(junit,), + readable_roots=(root,), ) if result.returncode == 124: return RunnerExecution( @@ -757,6 +760,7 @@ def _collect_node_ids( env=_pytest_environment(home) | { "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), }, writable_roots=(home.parent,), writable_files=(collection_output,), + readable_roots=(root,), ) if result.returncode == 124: return ( diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 393b6b026..24b5bb788 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -148,6 +148,7 @@ def _live_processes(pids: set[int]) -> set[int]: def _sandbox_command( command: list[str], writable_roots: tuple[Path, ...], *, cwd: Path | None = None, writable_files: tuple[Path, ...] = (), limits: SupervisorLimits = SupervisorLimits(), + readable_roots: tuple[Path, ...] = (), ) -> tuple[list[str], Path | None]: """Return an explicitly detected macOS/Linux sandbox command.""" if sys.platform == "darwin": @@ -182,6 +183,8 @@ def bind(option: str, source: Path) -> None: argv.extend((option, f"@FD:{len(sources) - 1}@", str(source))) for item in _runtime_roots(command, workdir): bind("--ro-bind", item) + for item in readable_roots: + bind("--ro-bind", item.resolve()) argv.extend(("--dev", "/dev")) for item in writable_roots: bind("--bind", item.resolve()) @@ -201,13 +204,14 @@ def run_supervised( command: list[str], *, cwd: Path, timeout: int, env: dict[str, str], writable_roots: tuple[Path, ...], writable_files: tuple[Path, ...] = (), limits: SupervisorLimits = SupervisorLimits(), + readable_roots: tuple[Path, ...] = (), ) -> tuple[subprocess.CompletedProcess[str], bool]: """Run sandboxed and terminate marked descendants across session changes.""" # pylint: disable=consider-using-with,too-many-locals,too-many-branches,too-many-statements try: argv, profile = _sandbox_command( command, writable_roots, cwd=cwd, writable_files=writable_files, - limits=limits, + limits=limits, readable_roots=readable_roots, ) except RuntimeError as exc: return subprocess.CompletedProcess(command, 125, "", str(exc)), False diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 3484ef45c..adb969576 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -67,7 +67,7 @@ def test_detached_session_descendant_is_terminated(tmp_path: Path) -> None: writable_roots=(tmp_path,), ) assert result.returncode == 0 - assert surviving is True + assert surviving is False time.sleep(1.2) assert not marker.exists() @@ -93,7 +93,7 @@ def test_detached_descendant_cannot_escape_by_removing_marker(tmp_path: Path) -> timeout=10, env=dict(os.environ), writable_roots=(tmp_path,), ) assert result.returncode == 0 - assert surviving is True + assert surviving is False time.sleep(1.2) assert not marker.exists() From ef3a5d4f9ef334a92a4673af2db75d9cc3a3479a Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 15:36:49 -0700 Subject: [PATCH 067/128] fix(sync): mount checker collection probe --- pdd/sync_core/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index ada6d90db..b5c6b1f8c 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -760,7 +760,7 @@ def _collect_node_ids( env=_pytest_environment(home) | { "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), }, writable_roots=(home.parent,), writable_files=(collection_output,), - readable_roots=(root,), + readable_roots=(root, _CHECKER_PYTEST_PROBE), ) if result.returncode == 124: return ( From 4fcba5038b9655f00e6d017d81f3e3ef52daeea8 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 15:39:43 -0700 Subject: [PATCH 068/128] test(sync): add focused protected runner CI lane --- .github/workflows/unit-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c158ded21..367d80452 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -89,6 +89,14 @@ jobs: assert not (root / "outside").exists() PY + - name: Run focused protected-runner tests + run: > + pytest -q + tests/test_sync_core_supervisor.py + tests/test_sync_core_runner.py + tests/test_sync_core_reporting.py + --timeout=60 + - name: Validate architecture vs prompt includes (fixture smoke) run: pdd checkup --validate-arch-includes --project-root tests/fixtures/arch_include_validate_ok From 13840cd5558cf2738f240779ecaa87ce1784a4b9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 16:28:20 -0700 Subject: [PATCH 069/128] test(sync): reproduce next review boundary gaps --- .github/workflows/unit-tests.yml | 1 + tests/test_sync_core_lifecycle_scenarios.py | 28 +++++++++ tests/test_sync_core_reporting.py | 19 ++++++ tests/test_sync_core_runner.py | 60 +++++++++++++++++++ tests/test_sync_core_supervisor.py | 52 +++++++++++++++- tests/test_sync_core_transaction.py | 21 +++++++ tests/test_sync_core_verification_profiles.py | 18 ++++++ tests/test_sync_determine_operation.py | 40 +++++++++++++ 8 files changed, 238 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 367d80452..b20c0eb6d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -94,6 +94,7 @@ jobs: pytest -q tests/test_sync_core_supervisor.py tests/test_sync_core_runner.py + tests/test_sync_core_lifecycle_scenarios.py tests/test_sync_core_reporting.py --timeout=60 diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index 85648269a..ba030b968 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -16,6 +16,7 @@ from pdd.sync_core.lifecycle import ( _install_candidate_wheel, _isolated_lifecycle_environment, + _lifecycle_command, run_lifecycle_matrix, ) from pdd.sync_core.scenario_contract import REQUIRED_SCENARIOS @@ -88,6 +89,33 @@ def _write_wheel( return wheel +def test_lifecycle_command_maps_inputs_read_only_and_environment_immutable( + tmp_path, monkeypatch +) -> None: + scratch = tmp_path / "scratch" + scratch.mkdir() + environment = tmp_path / "candidate-environment" + environment.mkdir() + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + cloud = tmp_path / "cloud" + cloud.mkdir() + captured = {} + + def fake_run(command, **kwargs): + captured.update(kwargs) + return subprocess.CompletedProcess(command, 0, "", ""), False + + monkeypatch.setattr("pdd.sync_core.lifecycle.run_supervised", fake_run) + result = _lifecycle_command( + [sys.executable, "-c", "pass"], scratch, scratch / "home", + readable_roots=(environment, wheelhouse, cloud), + ) + assert result.returncode == 0 + assert captured["writable_roots"] == (scratch,) + assert captured["readable_roots"] == (environment, wheelhouse, cloud) + + def test_lifecycle_contract_requires_public_repair_injection_scenarios() -> None: required = { "public-prompt-only-repair-zero-write-rerun", diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index 6e99cd701..18275a082 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -539,6 +539,25 @@ def test_trusted_finalizer_rejects_dirty_support_before_reuse(tmp_path) -> None: ) +def test_trusted_finalizer_rejects_allowed_state_renamed_to_support(tmp_path) -> None: + root, commit = _repository(tmp_path) + replay = tmp_path / "external-trust/rename-reuse.json" + finalize_unit( + root, PurePosixPath("prompts/widget_python.prompt"), base_ref=commit, + head_ref=commit, signer=SIGNER, replay_ledger_path=replay, + ) + evidence = next((root / ".pdd/evidence/v2").glob("*.json")) + subprocess.run(["git", "add", evidence], cwd=root, check=True) + subprocess.run(["git", "commit", "-q", "-m", "durable evidence"], cwd=root, check=True) + evidence.rename(root / "conftest.py") + with pytest.raises(ValueError, match="completely clean checkout"): + finalize_unit( + root, PurePosixPath("prompts/widget_python.prompt"), base_ref=commit, + head_ref=_git(root, "rev-parse", "HEAD"), signer=SIGNER, + replay_ledger_path=replay, + ) + + @pytest.mark.parametrize( ("edits", "semantic", "changed_roles"), [ diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 9041e9f02..86f432b28 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -23,6 +23,7 @@ _has_dynamic_pytest_plugins, _local_module_paths, pytest_validator_config_digest, + runner_identity_digest, ) @@ -772,3 +773,62 @@ def test_managed_subprocess_fails_closed_without_supported_os_sandbox( assert "unsupported sandbox platform" in result.stderr assert "must not execute" not in result.stdout assert surviving is False + + +@pytest.mark.parametrize( + "source", + [ + "pytest_plugins = []\npytest_plugins += ['vendor.runtime']\n", + "pytest_plugins = []\n(alias := pytest_plugins).append('vendor.runtime')\n", + "pytest_plugins = ['tests.local']\ndel pytest_plugins[0]\n", + "pytest_plugins = ['tests.local']\npytest_plugins[0] = 'vendor.runtime'\n", + "pytest_plugins = []\nalias = pytest_plugins\nalias.value = 'vendor.runtime'\n", + ], +) +def test_effective_pytest_plugin_mutations_fail_closed(source: str) -> None: + _resolved, dynamic = _local_module_paths( + Path("."), "HEAD", PurePosixPath("tests/conftest.py"), source.encode() + ) + assert dynamic is True + + +def test_runner_identity_binds_code_under_test_role_policy(tmp_path: Path) -> None: + root, commit = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "product.py").write_text("VALUE = 1\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "product") + commit = _git(root, "rev-parse", "HEAD") + support = _profile(root, commit) + product = _profile(root, commit, (PurePosixPath("product.py"),)) + assert runner_identity_digest(support, root=root, ref=commit) != ( + runner_identity_digest(product, root=root, ref=commit) + ) + + +def test_candidate_leader_exit_cannot_forge_junit_pass(tmp_path: Path) -> None: + content = ( + "import os, sys\n" + "for arg in sys.argv:\n" + " if arg.startswith('--junitxml='):\n" + " open(arg.split('=', 1)[1], 'w').write(" + "'')\n" + "os._exit(0)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_candidate_leader_exit_cannot_forge_collection_pass(tmp_path: Path) -> None: + content = ( + "import os\n" + "output = os.environ.get('PDD_TRUSTED_COLLECTION_OUTPUT')\n" + "if output:\n" + " open(output, 'w').write('[\"tests/test_widget.py::test_widget\"]')\n" + " os._exit(0)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index adb969576..b33de80ea 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -10,7 +10,13 @@ import pytest -from pdd.sync_core.supervisor import SupervisorLimits, _sandbox_command, run_supervised +from pdd.sync_core.supervisor import ( + SupervisorLimits, + _limited_command, + _live_processes, + _sandbox_command, + run_supervised, +) def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( @@ -188,3 +194,47 @@ def test_macos_fails_closed_without_kernel_lifetime_containment( assert result.returncode == 125 assert "cannot prove process lifetime isolation" in result.stderr assert surviving is False + + +def test_file_size_limit_uses_output_budget() -> None: + limits = SupervisorLimits(max_output_bytes=1234, max_writable_bytes=987654) + command = _limited_command(["/bin/true"], limits) + assert "1234" in command + assert "987654" not in command[1:7] + + +@pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") +def test_binary_output_has_aggregate_limit_and_deterministic_decode(tmp_path: Path) -> None: + limits = SupervisorLimits(max_output_bytes=1024) + result, _surviving = run_supervised( + [sys.executable, "-c", "import os; os.write(1, b'\\xff' * 800); os.write(2, b'x' * 800)"], + cwd=tmp_path, timeout=10, env=dict(os.environ), + writable_roots=(tmp_path,), limits=limits, + ) + assert result.returncode == 125 + assert len(result.stdout.encode("utf-8")) + len(result.stderr.encode("utf-8")) <= 3 * 1024 + assert "\ufffd" in result.stdout + + +def test_live_processes_rejects_reused_pid_identity(monkeypatch) -> None: + monkeypatch.setattr("pdd.sync_core.supervisor._process_identity", lambda _pid: "new") + monkeypatch.setattr(os, "kill", lambda *_args: None) + assert _live_processes({123: "old"}) == set() + + +@pytest.mark.skipif(not shutil.which("bwrap"), reason="requires Linux bubblewrap") +def test_writable_churn_cannot_escape_supervisor_cleanup(tmp_path: Path) -> None: + program = ( + "import pathlib, threading, time\n" + "root=pathlib.Path('.')\n" + "def churn():\n" + " while True:\n" + " p=root/'churn'; p.write_bytes(b'x'); p.unlink(missing_ok=True)\n" + "threading.Thread(target=churn, daemon=True).start(); time.sleep(.5)\n" + ) + result, surviving = run_supervised( + [sys.executable, "-c", program], cwd=tmp_path, timeout=5, + env=dict(os.environ), writable_roots=(tmp_path,), + ) + assert result.returncode == 0 + assert surviving is False diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index 6020d5677..faef84dce 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -231,3 +231,24 @@ def swap_after_resolution(relpath): assert outside_target.read_text() == "outside = True\n" assert (tmp_path / "src-before-swap/widget.py").read_text() == "value = 1\n" assert not (tmp_path / ".pdd/evidence/widget.json").exists() + + +def test_prepare_failure_never_publishes_orphan_transaction(tmp_path, monkeypatch) -> None: + (tmp_path / "src").mkdir() + (tmp_path / "src/widget.py").write_text("value = 1\n") + manager = TransactionManager(tmp_path) + original = manager._write_blob # pylint: disable=protected-access + calls = 0 + + def fail_second(path, content): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("simulated prepare failure") + return original(path, content) + + monkeypatch.setattr(manager, "_write_blob", fail_second) + with pytest.raises(OSError, match="simulated prepare failure"): + manager.prepare("tx-failed", _writes()) + assert not (tmp_path / ".pdd/transactions/tx-failed").exists() + assert manager.incomplete() == () diff --git a/tests/test_sync_core_verification_profiles.py b/tests/test_sync_core_verification_profiles.py index 4ddc44a2f..4ad7f1930 100644 --- a/tests/test_sync_core_verification_profiles.py +++ b/tests/test_sync_core_verification_profiles.py @@ -169,3 +169,21 @@ def test_candidate_only_profile_cannot_approve_itself(tmp_path) -> None: profiles = load_verification_profiles(root, _manifest(root, base, head)) assert profiles.coverage == 0.0 assert any("lacks protected approval" in item for item in profiles.invalid_reasons) + + +def test_profile_digest_binds_code_under_test_role_policy(tmp_path) -> None: + root = _repository(tmp_path) + profile_path = root / ".pdd/verification-profiles.json" + support = _profile() + profile_path.write_text(json.dumps(support)) + base = _commit(root, "support role") + support_digest = load_verification_profiles(root, _manifest(root, base, base)).profiles[0].profile_digest + + product = _profile() + product["profiles"][0]["obligations"][0]["code_under_test_paths"] = ["src/widget.py"] + (root / "src").mkdir() + (root / "src/widget.py").write_text("VALUE = 1\n") + profile_path.write_text(json.dumps(product)) + head = _commit(root, "product role") + product_digest = load_verification_profiles(root, _manifest(root, head, head)).profiles[0].profile_digest + assert support_digest != product_digest diff --git a/tests/test_sync_determine_operation.py b/tests/test_sync_determine_operation.py index bd5217732..e11d9b1d0 100644 --- a/tests/test_sync_determine_operation.py +++ b/tests/test_sync_determine_operation.py @@ -5569,3 +5569,43 @@ def test_pdd_path_unset_generation_matches_sync_extension(self, tmp_path, monkey f"FM1: generation writes {written.suffix!r} but sync expects " f"{expected.suffix!r} (PDD_PATH unset) -> #551 regeneration loop" ) + + +def test_v1_hash_matches_base_whitespace_cwd_and_invalid_utf8(tmp_path, monkeypatch): + prompt_dir = tmp_path / "prompts" + cwd = tmp_path / "cwd" + prompt_dir.mkdir() + cwd.mkdir() + dependency = cwd / "shared.bin" + dependency.write_bytes(b"dependency\xff") + prompt = prompt_dir / "widget.prompt" + prompt.write_bytes(b" shared.bin \ninvalid:\xff\n") + monkeypatch.chdir(cwd) + expected = hashlib.sha256(prompt.read_bytes() + dependency.read_bytes()).hexdigest() + assert calculate_prompt_hash(prompt, hash_version=1) == expected + + +def test_v1_hash_resolves_stored_relative_keys_from_cwd(tmp_path, monkeypatch): + prompt_dir = tmp_path / "prompts" + cwd = tmp_path / "cwd" + prompt_dir.mkdir() + cwd.mkdir() + dependency = cwd / "stored.txt" + dependency.write_bytes(b"stored") + prompt = prompt_dir / "widget.prompt" + prompt.write_bytes(b"no includes\n") + monkeypatch.chdir(cwd) + expected = hashlib.sha256(prompt.read_bytes() + dependency.read_bytes()).hexdigest() + assert calculate_prompt_hash( + prompt, {"stored.txt": "ignored"}, hash_version=1 + ) == expected + + +def test_v1_old_grammar_ignores_self_closing_and_path_attributes(tmp_path): + prompt = tmp_path / "widget.prompt" + prompt.write_text( + '\nmissing.txt\n' + ) + assert calculate_prompt_hash(prompt, hash_version=1) == hashlib.sha256( + prompt.read_bytes() + ).hexdigest() From 57e6e482693b8ef2de70b6c811161e77768740e9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 16:33:54 -0700 Subject: [PATCH 070/128] fix(sync): close next review trust boundaries --- pdd/sync_core/finalize.py | 25 +++- pdd/sync_core/lifecycle.py | 33 ++++- pdd/sync_core/pytest_probe.py | 5 +- pdd/sync_core/runner.py | 54 ++++++- pdd/sync_core/supervisor.py | 149 ++++++++++---------- pdd/sync_core/transaction.py | 29 ++-- pdd/sync_core/verification.py | 3 + pdd/sync_determine_operation.py | 23 +-- tests/test_sync_core_lifecycle_scenarios.py | 3 + tests/test_sync_determine_operation.py | 4 +- 10 files changed, 213 insertions(+), 115 deletions(-) diff --git a/pdd/sync_core/finalize.py b/pdd/sync_core/finalize.py index a8a2512f9..106cd8784 100644 --- a/pdd/sync_core/finalize.py +++ b/pdd/sync_core/finalize.py @@ -217,8 +217,8 @@ def _reusable_result( def _checkout_is_clean_for_finalization(root: Path) -> bool: """Allow only checker-owned durable state from an earlier finalization.""" status = subprocess.run( - ["git", "status", "--porcelain", "--untracked-files=all"], - cwd=root, capture_output=True, text=True, check=False, + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], + cwd=root, capture_output=True, check=False, ) if status.returncode != 0: return False @@ -226,10 +226,23 @@ def _checkout_is_clean_for_finalization(root: Path) -> bool: ".pdd/meta/v2/", ".pdd/evidence/v2/", ".pdd/locks/transactions/", ".pdd/transactions/", ) - return all( - len(line) >= 4 and line[3:].replace('"', "").startswith(allowed) - for line in status.stdout.splitlines() - ) + fields = status.stdout.split(b"\0") + index = 0 + while index < len(fields) and fields[index]: + record = fields[index] + if len(record) < 4: + return False + code = record[:2] + paths = [record[3:].decode("utf-8", errors="surrogateescape")] + if b"R" in code or b"C" in code: + index += 1 + if index >= len(fields) or not fields[index]: + return False + paths.append(fields[index].decode("utf-8", errors="surrogateescape")) + if any(not path.startswith(allowed) for path in paths): + return False + index += 1 + return True def finalize_unit( diff --git a/pdd/sync_core/lifecycle.py b/pdd/sync_core/lifecycle.py index 759d4442b..d712ddcbe 100644 --- a/pdd/sync_core/lifecycle.py +++ b/pdd/sync_core/lifecycle.py @@ -147,11 +147,17 @@ def _candidate_interpreter_identity( def _lifecycle_command(command: list[str], temporary: Path, home: Path, - timeout: int = 1200) -> subprocess.CompletedProcess[str]: + timeout: int = 1200, *, + readable_roots: tuple[Path, ...] = (), + writable_roots: tuple[Path, ...] | None = None, + cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + # pylint: disable=too-many-arguments """Route every lifecycle command through the shared fail-closed supervisor.""" result, surviving = run_supervised( - command, cwd=temporary, timeout=timeout, - env=_isolated_lifecycle_environment(home), writable_roots=(temporary,) + command, cwd=cwd or temporary, timeout=timeout, + env=_isolated_lifecycle_environment(home), + writable_roots=writable_roots or (temporary,), + readable_roots=readable_roots, ) if surviving: return subprocess.CompletedProcess(command, 125, result.stdout, @@ -218,15 +224,21 @@ def _install_candidate_wheel( "-r", str(combined_lock), ], - temporary, home, + temporary, home, readable_roots=(wheelhouse, wheel, runtime_lock), ) if installed.returncode != 0: return None + closure = _installed_file_closure(candidate_python) + proof_scratch = temporary / "proof-scratch" + proof_scratch.mkdir(mode=0o700) proved = _lifecycle_command( - [str(candidate_python), "-I", "-m", "pdd.cli", "--help"], temporary, home + [str(candidate_python), "-I", "-m", "pdd.cli", "--help"], temporary, home, + readable_roots=(environment,), writable_roots=(proof_scratch,), cwd=proof_scratch, ) if proved.returncode != 0: return None + if _installed_file_closure(candidate_python) != closure: + return None dependency_digest = _dependency_environment_digest(candidate_python, isolated) if len(dependency_digest) != 64: return None @@ -305,6 +317,7 @@ def run_lifecycle_matrix( if installed_candidate is None: return _failed_result() candidate_python, dependency_digest = installed_candidate + installed_files = _installed_file_closure(candidate_python) measured_python = _candidate_interpreter_identity( candidate_python, _isolated_lifecycle_environment(temporary / "home") ) @@ -332,9 +345,15 @@ def run_lifecycle_matrix( "--candidate-python", str(candidate_python), ] + scenario_scratch = temporary / "scenario-scratch" + scenario_scratch.mkdir(mode=0o700) completed = _lifecycle_command( - command, temporary, temporary / "home", timeout_seconds + command, temporary, temporary / "home", timeout_seconds, + readable_roots=(candidate_python.parents[1], Path(cloud_root).resolve()), + writable_roots=(scenario_scratch,), cwd=scenario_scratch, ) + if _installed_file_closure(candidate_python) != installed_files: + return _failed_result() if completed.returncode == 124: return _failed_result(timeout=True) try: @@ -371,6 +390,6 @@ def run_lifecycle_matrix( candidate_artifact, protected_lock_digest, measured_python, - _installed_file_closure(candidate_python), + installed_files, "pdd-released-checker-v1", ) diff --git a/pdd/sync_core/pytest_probe.py b/pdd/sync_core/pytest_probe.py index 0807ec92d..665f49879 100644 --- a/pdd/sync_core/pytest_probe.py +++ b/pdd/sync_core/pytest_probe.py @@ -3,13 +3,12 @@ from __future__ import annotations import json -import os from pathlib import Path import pytest -_OUTPUT_ENV = "PDD_TRUSTED_COLLECTION_OUTPUT" +_OUTPUT_PATH: str | None = None @pytest.hookimpl(wrapper=True, tryfirst=True) @@ -19,7 +18,7 @@ def pytest_collection_modifyitems(items): try: return (yield) finally: - output = os.environ.get(_OUTPUT_ENV) + output = _OUTPUT_PATH if output: Path(output).write_text( json.dumps(protected_node_ids, separators=(",", ":")), diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index b5c6b1f8c..3fe32df8f 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -104,6 +104,10 @@ def _pytest_plugin_declaration_targets(node: ast.AST) -> tuple[ast.AST, ...]: return tuple(node.targets) if isinstance(node, ast.AnnAssign): return (node.target,) + if isinstance(node, (ast.AugAssign, ast.NamedExpr)): + return (node.target,) + if isinstance(node, ast.Delete): + return tuple(node.targets) return () @@ -136,7 +140,7 @@ def _local_module_paths( plugin_aliases = {"pytest_plugins"} for candidate in ast.walk(tree): if ( - isinstance(candidate, (ast.Assign, ast.AnnAssign)) + isinstance(candidate, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) and isinstance(candidate.value, ast.Name) and candidate.value.id in plugin_aliases ): @@ -146,6 +150,20 @@ def _local_module_paths( ) pytest_plugins_declared = False for node in ast.walk(tree): + targets = _pytest_plugin_declaration_targets(node) + def rooted_in_plugin(target: ast.AST) -> bool: + while isinstance(target, (ast.Subscript, ast.Attribute)): + target = target.value + return isinstance(target, ast.Name) and target.id in plugin_aliases + if isinstance(node, (ast.AugAssign, ast.NamedExpr, ast.Delete)) and any( + rooted_in_plugin(target) for target in targets + ): + dynamic_pytest_plugins = True + elif isinstance(node, (ast.Assign, ast.AnnAssign)) and any( + isinstance(target, (ast.Subscript, ast.Attribute)) + and rooted_in_plugin(target) for target in targets + ): + dynamic_pytest_plugins = True if isinstance(node, ast.Import): modules.update(alias.name for alias in node.names) importlib_names.update( @@ -511,6 +529,9 @@ def runner_identity_digest( "validator": item.validator_id, "config": item.validator_config_digest, "paths": [path.as_posix() for path in item.artifact_paths], + "code_under_test_paths": [ + path.as_posix() for path in sorted(item.code_under_test_paths) + ], } for item in profile.obligations ], @@ -634,6 +655,7 @@ def _trusted_collection_runner( directory: Path, root: Path, pytest_args: list[str], + collection_output: Path, ) -> Path: """Create a checker-owned pytest entrypoint that imports the probe by path.""" runner = directory / "run_collection.py" @@ -657,6 +679,7 @@ def _trusted_collection_runner( "_MODULE = importlib.util.module_from_spec(_SPEC)", "sys.modules['_pdd_checker_pytest_probe_abs'] = _MODULE", "_SPEC.loader.exec_module(_MODULE)", + f"_MODULE._OUTPUT_PATH = {json.dumps(str(collection_output))}", "", "import pytest", "if _ROOT not in sys.path:", @@ -670,6 +693,23 @@ def _trusted_collection_runner( return runner +def _trusted_execution_runner( + directory: Path, root: Path, pytest_args: list[str], junit: Path +) -> Path: + """Create a controller entrypoint that owns the hidden result destination.""" + runner = directory / "run_execution.py" + runner.write_text( + "\n".join(( + "import os, sys", "import pytest", + f"os.chdir({json.dumps(str(root))})", + f"_ARGS = {json.dumps(pytest_args + [f'--junitxml={junit}'])}", + f"sys.path.insert(0, {json.dumps(str(root))})", + "raise SystemExit(pytest.main(_ARGS))", "", + )), encoding="utf-8", + ) + return runner + + def _run_test_node( root: Path, node_id: str, @@ -692,8 +732,9 @@ def _run_test_node( home.mkdir(mode=0o700, parents=True) junit = temporary / "junit.xml" junit.touch(mode=0o600) + controller = _trusted_execution_runner(temporary, root, command[3:], junit) result, surviving = _managed_subprocess( - [*command, f"--junitxml={junit}"], cwd=root, + [sys.executable, str(controller)], cwd=temporary, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), writable_files=(junit,), readable_roots=(root,), @@ -741,7 +782,9 @@ def _collect_node_ids( "no:cacheprovider", path.as_posix(), ] - runner = _trusted_collection_runner(temporary, root, pytest_args) + runner = _trusted_collection_runner( + temporary, root, pytest_args, collection_output + ) command = [sys.executable, str(runner)] digest = hashlib.sha256( json.dumps( @@ -757,9 +800,8 @@ def _collect_node_ids( ).hexdigest() result, surviving = _managed_subprocess( command, cwd=temporary, timeout=timeout_seconds, - env=_pytest_environment(home) | { - "PDD_TRUSTED_COLLECTION_OUTPUT": str(collection_output), - }, writable_roots=(home.parent,), writable_files=(collection_output,), + env=_pytest_environment(home), writable_roots=(home.parent,), + writable_files=(collection_output,), readable_roots=(root, _CHECKER_PYTEST_PROBE), ) if result.returncode == 124: diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 24b5bb788..ab53260ea 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -54,7 +54,7 @@ def _limited_command(command: list[str], limits: SupervisorLimits) -> list[str]: ) return [sys.executable, "-c", script, str(limits.max_memory_bytes), str(limits.max_cpu_seconds), str(limits.max_processes), - str(limits.max_writable_bytes), "256", *command] + str(limits.max_output_bytes), "256", *command] def _staged_bwrap(argv: list[str], sources: list[Path]) -> list[str]: @@ -133,10 +133,25 @@ def _process_descendants(root_pid: int) -> set[int]: return found -def _live_processes(pids: set[int]) -> set[int]: - """Filter historical observations to processes that still exist.""" +def _process_identity(pid: int) -> str | None: + """Return a stable Linux process identity, including kernel start time.""" + if sys.platform.startswith("linux"): + try: + stat_fields = Path(f"/proc/{pid}/stat").read_text( + encoding="utf-8" + ).rsplit(")", 1)[1].split() + return stat_fields[19] + except (OSError, IndexError): + return None + return None + + +def _live_processes(pids: dict[int, str | None]) -> set[int]: + """Return only observations whose stable process identity still matches.""" live = set() - for pid in pids: + for pid, identity in pids.items(): + if identity is None or _process_identity(pid) != identity: + continue try: os.kill(pid, 0) except (ProcessLookupError, PermissionError): @@ -145,6 +160,23 @@ def _live_processes(pids: set[int]) -> set[int]: return live +def _writable_size(roots: tuple[Path, ...]) -> int: + """Measure a concurrently mutable tree without allowing races to escape.""" + total = 0 + for root in roots: + try: + paths = root.rglob("*") + for path in paths: + try: + if path.is_file() and not path.is_symlink(): + total += path.stat().st_size + except OSError: + continue + except OSError: + continue + return total + + def _sandbox_command( command: list[str], writable_roots: tuple[Path, ...], *, cwd: Path | None = None, writable_files: tuple[Path, ...] = (), limits: SupervisorLimits = SupervisorLimits(), @@ -216,10 +248,10 @@ def run_supervised( except RuntimeError as exc: return subprocess.CompletedProcess(command, 125, "", str(exc)), False token = uuid.uuid4().hex - stdout_file = tempfile.TemporaryFile(mode="w+", encoding="utf-8") - stderr_file = tempfile.TemporaryFile(mode="w+", encoding="utf-8") + stdout_file = tempfile.TemporaryFile(mode="w+b") + stderr_file = tempfile.TemporaryFile(mode="w+b") process = subprocess.Popen( - argv, cwd=cwd, stdout=stdout_file, stderr=stderr_file, text=True, + argv, cwd=cwd, stdout=stdout_file, stderr=stderr_file, env=env | {"PYTHONDONTWRITEBYTECODE": "1", "PDD_SUPERVISION_TOKEN": token, "TMPDIR": str(writable_roots[0].resolve()), @@ -229,89 +261,64 @@ def run_supervised( ) timed_out = False surviving = False - tracked: set[int] = set() + tracked: dict[int, str | None] = {} tracking_done = threading.Event() def track_process_tree() -> None: while not tracking_done.wait(0.005): - tracked.update(_process_descendants(process.pid)) + for pid in _process_descendants(process.pid): + tracked.setdefault(pid, _process_identity(pid)) tracker = threading.Thread(target=track_process_tree, daemon=True) tracker.start() deadline = time.monotonic() + timeout output_limited = False - while True: - try: - process.communicate( - timeout=max(0.01, min(0.1, deadline - time.monotonic())) - ) - break - except subprocess.TimeoutExpired: - writable_size = sum( - path.stat().st_size for root in writable_roots - for path in root.rglob("*") if path.is_file() - ) - if writable_size > limits.max_writable_bytes: - output_limited = True - os.killpg(process.pid, signal.SIGKILL) - process.communicate() - break - descendants = _supervised_descendants(token) - {process.pid} - if process.poll() is not None and descendants: - surviving = True - for pid in descendants: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass + try: + while process.poll() is None: if time.monotonic() >= deadline: timed_out = True - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - for pid in _supervised_descendants(token) - {process.pid}: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.communicate() break - tracking_done.set() - tracker.join(timeout=1) - if not timed_out and os.name != "nt": - try: - os.killpg(process.pid, 0) - surviving = True - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - descendants = _live_processes( - (_supervised_descendants(token) | tracked) - {process.pid} - ) - if descendants: - surviving = True - for pid in descendants: + if _writable_size(writable_roots) > limits.max_writable_bytes: + output_limited = True + break + if stdout_file.tell() + stderr_file.tell() > limits.max_output_bytes: + output_limited = True + break + time.sleep(0.01) + finally: + tracking_done.set() + tracker.join(timeout=1) + if process.poll() is None: try: - os.kill(pid, signal.SIGKILL) + os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass - if profile is not None: - profile.unlink(missing_ok=True) + process.wait() + observed = _supervised_descendants(token) - {process.pid} + for pid in observed: + tracked.setdefault(pid, _process_identity(pid)) + descendants = _live_processes(tracked) + if descendants: + surviving = True + for pid in descendants: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if profile is not None: + profile.unlink(missing_ok=True) stdout_file.seek(0) stderr_file.seek(0) - stdout = stdout_file.read(limits.max_output_bytes + 1) - stderr = stderr_file.read(limits.max_output_bytes + 1) + remaining = limits.max_output_bytes + stdout_bytes = stdout_file.read(remaining) + remaining -= len(stdout_bytes) + stderr_bytes = stderr_file.read(remaining) + if stdout_file.read(1) or stderr_file.read(1): + output_limited = True stdout_file.close() stderr_file.close() - encoded = stdout.encode() - if len(encoded) > limits.max_output_bytes: - stdout = encoded[:limits.max_output_bytes].decode("utf-8", errors="replace") - output_limited = True - encoded = stderr.encode() - if len(encoded) > limits.max_output_bytes: - stderr = encoded[:limits.max_output_bytes].decode("utf-8", errors="replace") - output_limited = True + stdout = stdout_bytes.decode("utf-8", errors="replace") + stderr = stderr_bytes.decode("utf-8", errors="replace") return subprocess.CompletedProcess( command, 125 if output_limited else (124 if timed_out else process.returncode), stdout, stderr, diff --git a/pdd/sync_core/transaction.py b/pdd/sync_core/transaction.py index d5a8c7659..6a48d63f6 100644 --- a/pdd/sync_core/transaction.py +++ b/pdd/sync_core/transaction.py @@ -391,16 +391,23 @@ def prepare( transaction_dir = self._transaction_dir(transaction_id) if transaction_dir.exists(): raise TransactionError(f"transaction already exists: {transaction_id}") - transaction_dir.mkdir(mode=0o700) - entries = [self._entry(transaction_dir, index, write) for index, write in enumerate(writes)] - payload: dict[str, object] = { - "schema_version": 1, - "transaction_id": transaction_id, - "phase": TransactionPhase.PREPARED.value, - "shared_resources": [path.as_posix() for path in sorted(shared_resources)], - "entries": entries, - } - self._write_journal(transaction_dir, payload) + temporary = Path(tempfile.mkdtemp(prefix=".prepare-", dir=self.state_root)) + temporary.chmod(0o700) + try: + entries = [self._entry(temporary, index, write) for index, write in enumerate(writes)] + payload: dict[str, object] = { + "schema_version": 1, + "transaction_id": transaction_id, + "phase": TransactionPhase.PREPARED.value, + "shared_resources": [path.as_posix() for path in sorted(shared_resources)], + "entries": entries, + } + self._write_journal(temporary, payload) + fsync_directory(temporary) + os.replace(temporary, transaction_dir) + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise fsync_directory(self.state_root) return TransactionResult( transaction_id, @@ -570,6 +577,8 @@ def incomplete(self) -> tuple[str, ...]: for transaction_dir in sorted(self.state_root.iterdir()): if not transaction_dir.is_dir() or transaction_dir.is_symlink(): continue + if transaction_dir.name.startswith(".prepare-"): + continue phase = TransactionPhase(str(self._read_journal(transaction_dir)["phase"])) if phase not in {TransactionPhase.COMMITTED, TransactionPhase.ROLLED_BACK}: pending.append(transaction_dir.name) diff --git a/pdd/sync_core/verification.py b/pdd/sync_core/verification.py index e2edeea94..e8d04a2bd 100644 --- a/pdd/sync_core/verification.py +++ b/pdd/sync_core/verification.py @@ -182,6 +182,9 @@ def _profile_digest( "validator_config_digest": item.validator_config_digest, "requirement_ids": item.requirement_ids, "artifact_paths": [path.as_posix() for path in item.artifact_paths], + "code_under_test_paths": [ + path.as_posix() for path in sorted(item.code_under_test_paths) + ], "required": item.required, } for item in obligations diff --git a/pdd/sync_determine_operation.py b/pdd/sync_determine_operation.py index 5368b7e0a..8fcc162e5 100644 --- a/pdd/sync_determine_operation.py +++ b/pdd/sync_determine_operation.py @@ -1781,13 +1781,12 @@ def extract_include_deps(prompt_path: Path) -> Dict[str, str]: def _legacy_dependency_path(prompt_path: Path, declared: str) -> Optional[Path]: - """Resolve a legacy include from prompt ancestry, never the process CWD.""" + """Resolve a legacy include exactly as the unversioned base implementation.""" candidate = Path(declared) - if candidate.is_absolute(): - return candidate if candidate.is_file() else None - for parent in (prompt_path.parent, *prompt_path.parent.parents): - resolved = parent / candidate - if resolved.is_file(): + if candidate.is_absolute() and candidate.exists(): + return candidate + for resolved in (prompt_path.parent / declared, Path.cwd() / declared): + if resolved.exists(): return resolved return None @@ -1823,8 +1822,8 @@ def calculate_prompt_hash( return None try: - prompt_content = prompt_path.read_text(encoding="utf-8") - except (IOError, OSError, UnicodeDecodeError): + prompt_content = prompt_path.read_text(encoding="utf-8", errors="ignore") + except (IOError, OSError): return None from pdd.sync_core.includes import parse_include_references @@ -1838,10 +1837,14 @@ def calculate_prompt_hash( if references else list((stored_deps or {}).keys()) ) if hash_version == 1: - declared_dependencies = sorted(set(declared_dependencies)) + declared_dependencies = sorted(set(item.strip() for item in declared_dependencies)) resolved_dependencies = [] for declared in declared_dependencies: - candidate = _legacy_dependency_path(prompt_path.resolve(), declared) + if hash_version == 1 and not references: + candidate = Path(declared) + candidate = candidate if candidate.exists() else None + else: + candidate = _legacy_dependency_path(prompt_path, declared) if candidate is None: if hash_version == 1: continue diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index ba030b968..0762a3b69 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -321,6 +321,9 @@ def fake_run(command, **_kwargs): ) +@pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="protected lifecycle requires Linux" +) def test_candidate_install_e2e_uses_locked_runtime_wheelhouse(tmp_path) -> None: wheelhouse = tmp_path / "wheelhouse" wheelhouse.mkdir() diff --git a/tests/test_sync_determine_operation.py b/tests/test_sync_determine_operation.py index e11d9b1d0..eca092726 100644 --- a/tests/test_sync_determine_operation.py +++ b/tests/test_sync_determine_operation.py @@ -4187,7 +4187,7 @@ def test_calculate_prompt_hash_with_stored_deps(self, pdd_test_environment): "stored deps should contribute to the composite hash" ) - def test_legacy_include_hash_matches_pre_versioned_bytes_across_cwds( + def test_legacy_include_hash_uses_prompt_dir_then_process_cwd( self, pdd_test_environment, monkeypatch ): prompt = pdd_test_environment / "prompts" / f"{BASENAME}_{LANGUAGE}.prompt" @@ -4199,7 +4199,7 @@ def test_legacy_include_hash_matches_pre_versioned_bytes_across_cwds( for cwd in (pdd_test_environment, pdd_test_environment.parent): monkeypatch.chdir(cwd) hashes.append(calculate_prompt_hash(prompt)) - assert hashes == [expected, expected] + assert hashes == [expected, hashlib.sha256(prompt.read_bytes()).hexdigest()] def test_legacy_include_hash_sorts_and_deduplicates_dependencies( self, pdd_test_environment From f4ffd0521b858d72ef889360ec5547f3ddb22137 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 17:28:07 -0700 Subject: [PATCH 071/128] test(sync): reproduce five review findings --- tests/test_sync_core_includes.py | 14 +++---- tests/test_sync_core_runner.py | 56 ++++++++++++++++++++++++++ tests/test_sync_determine_operation.py | 24 +++++++++++ tests/test_update_main.py | 46 +++++++++++++++++++++ 4 files changed, 132 insertions(+), 8 deletions(-) diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py index e843c67ed..a258de0da 100644 --- a/tests/test_sync_core_includes.py +++ b/tests/test_sync_core_includes.py @@ -269,20 +269,18 @@ def test_manifest_output_alias_resolves_generated_code_relative_include(tmp_path ) -def test_parent_prefixed_repository_path_stays_inside_checkout(tmp_path) -> None: +def test_parent_prefixed_repository_path_is_rejected_even_with_same_tail(tmp_path) -> None: prompt = tmp_path / "prompts/frontend/components/widget.prompt" dependency = tmp_path / "frontend/src/context/AuthContext.tsx" prompt.parent.mkdir(parents=True) dependency.parent.mkdir(parents=True) prompt.write_text("../../frontend/src/context/AuthContext.tsx") dependency.write_text("export const AuthContext = {};\n") - closure = build_include_closure( - PurePosixPath("prompts/frontend/components/widget.prompt"), - PathPolicy(tmp_path), - ) - assert closure.artifacts[0].relpath == PurePosixPath( - "frontend/src/context/AuthContext.tsx" - ) + with pytest.raises(IncludeGraphError, match="escapes repository"): + build_include_closure( + PurePosixPath("prompts/frontend/components/widget.prompt"), + PathPolicy(tmp_path), + ) def test_unresolved_external_package_forces_nondeterministic_closure(tmp_path) -> None: diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 86f432b28..abcef7fe2 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -20,6 +20,7 @@ run_profile, ) from pdd.sync_core.runner import ( + _config_loads_plugin, _has_dynamic_pytest_plugins, _local_module_paths, pytest_validator_config_digest, @@ -832,3 +833,58 @@ def test_candidate_leader_exit_cannot_forge_collection_pass(tmp_path: Path) -> N root, commit = _repository(tmp_path, content) _envelope, executions = _run(root, commit, commit) assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_candidate_controller_globals_cannot_forge_execution_pass(tmp_path: Path) -> None: + content = ( + "import os, sys\n" + "main = sys.modules['__main__']\n" + "for arg in getattr(main, '_ARGS', ()):\n" + " if arg.startswith('--junitxml='):\n" + " open(arg.split('=', 1)[1], 'w').write(" + "'')\n" + " os._exit(0)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_candidate_controller_globals_cannot_forge_collection_pass(tmp_path: Path) -> None: + content = ( + "import os, sys\n" + "main = sys.modules['__main__']\n" + "module = getattr(main, '_MODULE', None)\n" + "output = getattr(module, '_OUTPUT_PATH', None)\n" + "if output:\n" + " open(output, 'w').write('[\"tests/test_widget.py::test_widget\"]')\n" + " os._exit(0)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_pytest_plugin_guard_ignores_non_pytest_preview_prose(tmp_path: Path) -> None: + root, _commit = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "setup.cfg").write_text( + "[metadata]\ndescription = a -preview build\n", encoding="utf-8" + ) + _git(root, "add", "setup.cfg") + _git(root, "commit", "-q", "-m", "config prose") + assert not _config_loads_plugin(root, _git(root, "rev-parse", "HEAD")) + + +@pytest.mark.parametrize("option", ["-p local_plugin", "-plocal_plugin", "-p=local_plugin"]) +def test_pytest_plugin_guard_rejects_structured_pytest_addopts( + tmp_path: Path, option: str +) -> None: + root, _commit = _repository(tmp_path, "def test_widget(): assert True\n") + (root / "setup.cfg").write_text( + f"[tool:pytest]\naddopts = {option}\n", encoding="utf-8" + ) + _git(root, "add", "setup.cfg") + _git(root, "commit", "-q", "-m", "pytest plugin") + assert _config_loads_plugin(root, _git(root, "rev-parse", "HEAD")) diff --git a/tests/test_sync_determine_operation.py b/tests/test_sync_determine_operation.py index eca092726..96c0c6033 100644 --- a/tests/test_sync_determine_operation.py +++ b/tests/test_sync_determine_operation.py @@ -5609,3 +5609,27 @@ def test_v1_old_grammar_ignores_self_closing_and_path_attributes(tmp_path): assert calculate_prompt_hash(prompt, hash_version=1) == hashlib.sha256( prompt.read_bytes() ).hexdigest() + + +@pytest.mark.parametrize( + "markup", + [ + '', + 'dep.txt', + ], +) +def test_v1_new_grammar_save_reload_rerun_does_not_self_drift( + tmp_path, monkeypatch, markup +): + monkeypatch.chdir(tmp_path) + prompt = tmp_path / "widget.prompt" + prompt.write_text(markup, encoding="utf-8") + (tmp_path / "dep.txt").write_text("dependency", encoding="utf-8") + + first = calculate_prompt_hash(prompt, hash_version=1) + persisted = extract_include_deps(prompt, version=1) + reloaded = json.loads(json.dumps(persisted)) + second = calculate_prompt_hash(prompt, reloaded, hash_version=1) + + assert persisted == {} + assert second == first diff --git a/tests/test_update_main.py b/tests/test_update_main.py index 99d8ceea6..0e3e32eeb 100644 --- a/tests/test_update_main.py +++ b/tests/test_update_main.py @@ -6,6 +6,7 @@ import click from click.testing import CliRunner import git +import subprocess from pdd import DEFAULT_STRENGTH from pdd.update_main import ( @@ -15,6 +16,51 @@ update_main, ) + +def test_protected_update_cli_exits_nonzero_without_model_calls_or_writes( + tmp_path, monkeypatch +): + from pdd.cli import cli + + prompt = tmp_path / "prompts/widget_python.prompt" + modified = tmp_path / "widget.py" + original = tmp_path / "widget_original.py" + prompt.parent.mkdir() + prompt.write_text("original prompt\n", encoding="utf-8") + modified.write_text("VALUE = 2\n", encoding="utf-8") + original.write_text("VALUE = 1\n", encoding="utf-8") + (tmp_path / ".pdd").mkdir() + (tmp_path / ".pdd/repository-id").write_text("test-repository\n", encoding="utf-8") + (tmp_path / ".pdd/sync-policy.json").write_text( + '{"schema_version": 1, "enforcement": "active"}\n', encoding="utf-8" + ) + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run( + ["git", "config", "user.email", "update@example.com"], cwd=tmp_path, check=True + ) + subprocess.run( + ["git", "config", "user.name", "Update Test"], cwd=tmp_path, check=True + ) + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-q", "-m", "protected"], cwd=tmp_path, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=tmp_path, check=True, + capture_output=True, text=True, + ).stdout.strip() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("PDD_SYNC_PROTECTED_BASE_SHA", head) + before = {path: path.read_bytes() for path in (prompt, modified, original)} + + with patch("pdd.update_main.update_prompt") as model_call: + result = CliRunner().invoke( + cli, ["--quiet", "update", str(prompt), str(modified), str(original)] + ) + + assert result.exit_code != 0 + assert "protected canonical sync blocks legacy production mutation" in result.output + model_call.assert_not_called() + assert {path: path.read_bytes() for path in before} == before + @pytest.fixture def mock_ctx(): """ From aed3301bb2dc9f2fd68102959c5e1b751386d6e1 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 17:33:05 -0700 Subject: [PATCH 072/128] fix(sync): close five verification boundaries --- pdd/auto_deps_main.py | 2 + pdd/sync_core/finalize.py | 10 ++++ pdd/sync_core/includes.py | 11 +--- pdd/sync_core/runner.py | 95 ++++++++++++++++++++++---------- pdd/sync_determine_operation.py | 13 +++-- pdd/sync_main.py | 12 ++-- pdd/update_main.py | 16 ++++++ tests/test_sync_core_includes.py | 4 +- tests/test_update_main.py | 2 +- 9 files changed, 113 insertions(+), 52 deletions(-) diff --git a/pdd/auto_deps_main.py b/pdd/auto_deps_main.py index 94a96a48f..d862dda98 100644 --- a/pdd/auto_deps_main.py +++ b/pdd/auto_deps_main.py @@ -48,6 +48,8 @@ def auto_deps_main( ("", 0.0, f"Error: {exc}") on non-Abort failures so orchestrators can continue gracefully. """ + from .sync_core.finalize import preflight_legacy_mutation + preflight_legacy_mutation({"prompt": Path(prompt_file)}) console = Console() quiet = ctx.obj.get("quiet", False) if ctx.obj else False diff --git a/pdd/sync_core/finalize.py b/pdd/sync_core/finalize.py index 106cd8784..3f60696f7 100644 --- a/pdd/sync_core/finalize.py +++ b/pdd/sync_core/finalize.py @@ -60,6 +60,16 @@ class CanonicalFinalizationError(RuntimeError): """Raised when an opted-in mutation cannot commit trusted final state.""" +def preflight_legacy_mutation(paths: dict[str, Path] | None = None) -> None: + """Reject legacy production mutation before any model call or write.""" + if canonical_root_for_paths(paths) is not None: + raise CanonicalFinalizationError( + "protected canonical sync blocks legacy production mutation; " + "use read-only reporting or trusted finalization until the staged " + "repair executor is enabled" + ) + + def canonical_root_for_paths(paths: dict[str, Path] | None) -> Path | None: """Return the opted-in Git root for legacy path-based callers.""" # pylint: disable=import-outside-toplevel diff --git a/pdd/sync_core/includes.py b/pdd/sync_core/includes.py index e3c6667d3..b457cd450 100644 --- a/pdd/sync_core/includes.py +++ b/pdd/sync_core/includes.py @@ -334,17 +334,10 @@ def _candidate_paths( declared = PurePosixPath(raw_path) if declared.is_absolute(): raise IncludeGraphError(f"absolute include path is not allowed: {raw_path}") + if declared.parts and declared.parts[0] == "..": + _normalized(source.parent / declared, raw_path) candidates = [source.parent / declared] candidates.extend(alias.parent / declared for alias in aliases) - leading_parents = 0 - for part in declared.parts: - if part != "..": - break - leading_parents += 1 - if leading_parents: - repository_tail = PurePosixPath(*declared.parts[leading_parents:]) - if repository_tail.parts: - candidates.append(repository_tail) prompt_index = ( max(index for index, part in enumerate(source.parts) if part == "prompts") if "prompts" in source.parts diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 3fe32df8f..75ff69010 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -6,11 +6,13 @@ import hashlib import ast +import configparser import json import shlex import subprocess import sys import tempfile +import tomllib import xml.etree.ElementTree as ET from dataclasses import dataclass from datetime import datetime @@ -469,17 +471,32 @@ def _config_loads_plugin(root: Path, ref: str) -> bool: continue try: text = content.decode("utf-8") - except UnicodeDecodeError: - return True - try: - tokens = shlex.split(text.replace("=", " = "), comments=True) - except ValueError: + if path.name == "pyproject.toml": + pytest_options = tomllib.loads(text).get("tool", {}).get("pytest", {}).get( + "ini_options", {} + ) + values = [pytest_options.get("addopts", "")] + else: + parser = configparser.ConfigParser(interpolation=None) + parser.read_string(text) + section = "pytest" if path.name in {"pytest.ini", ".pytest.ini"} else "tool:pytest" + values = [parser.get(section, "addopts", fallback="")] + except (UnicodeDecodeError, ValueError, configparser.Error, AttributeError): return True - for index, token in enumerate(tokens): - if token == "-p" and index + 1 < len(tokens): + for value in values: + if isinstance(value, list): + value = " ".join(str(item) for item in value) + if not isinstance(value, str): return True - if token.startswith("-p=") or (token.startswith("-p") and len(token) > 2): + try: + tokens = shlex.split(value, comments=True) + except ValueError: return True + for index, token in enumerate(tokens): + if token == "-p" and index + 1 < len(tokens): + return True + if token.startswith("-p=") or (token.startswith("-p") and len(token) > 2): + return True return False @@ -656,10 +673,11 @@ def _trusted_collection_runner( root: Path, pytest_args: list[str], collection_output: Path, -) -> Path: - """Create a checker-owned pytest entrypoint that imports the probe by path.""" - runner = directory / "run_collection.py" - runner.write_text( +) -> tuple[Path, Path]: + """Create a process-separated checker and candidate collection worker.""" + worker = directory / "collection_worker.py" + worker_output = directory / "candidate-node-ids.json" + worker.write_text( "\n".join( ( "import importlib.util", @@ -668,7 +686,6 @@ def _trusted_collection_runner( "", f"_ROOT = {json.dumps(str(root))}", f"_PROBE = {json.dumps(str(_CHECKER_PYTEST_PROBE))}", - f"_ARGS = {json.dumps(pytest_args)}", "", "os.chdir(_ROOT)", "_SPEC = importlib.util.spec_from_file_location(", @@ -677,37 +694,57 @@ def _trusted_collection_runner( "if _SPEC is None or _SPEC.loader is None:", " raise ImportError('checker pytest probe is unavailable')", "_MODULE = importlib.util.module_from_spec(_SPEC)", - "sys.modules['_pdd_checker_pytest_probe_abs'] = _MODULE", "_SPEC.loader.exec_module(_MODULE)", - f"_MODULE._OUTPUT_PATH = {json.dumps(str(collection_output))}", + f"_MODULE._OUTPUT_PATH = {json.dumps(str(worker_output))}", "", "import pytest", "if _ROOT not in sys.path:", " sys.path.insert(0, _ROOT)", - "raise SystemExit(pytest.main(_ARGS, plugins=[_MODULE]))", + "raise SystemExit(pytest.main(" + json.dumps(pytest_args) + ", plugins=[_MODULE]))", "", ) ), encoding="utf-8", ) - return runner + checker = directory / "run_collection.py" + checker.write_text( + "\n".join(( + "import pathlib, subprocess, sys", + f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", + f"source = pathlib.Path({json.dumps(str(worker_output))})", + "if result.returncode != 0 or not source.is_file(): raise SystemExit(1)", + f"pathlib.Path({json.dumps(str(collection_output))}).write_bytes(source.read_bytes())", + "",)), encoding="utf-8", + ) + return checker, worker_output def _trusted_execution_runner( directory: Path, root: Path, pytest_args: list[str], junit: Path -) -> Path: - """Create a controller entrypoint that owns the hidden result destination.""" - runner = directory / "run_execution.py" - runner.write_text( +) -> tuple[Path, Path]: + """Create a checker that never imports candidate code or pytest.""" + worker = directory / "execution_worker.py" + worker_junit = directory / "candidate-junit.xml" + worker.write_text( "\n".join(( "import os, sys", "import pytest", f"os.chdir({json.dumps(str(root))})", - f"_ARGS = {json.dumps(pytest_args + [f'--junitxml={junit}'])}", f"sys.path.insert(0, {json.dumps(str(root))})", - "raise SystemExit(pytest.main(_ARGS))", "", + "raise SystemExit(pytest.main(" + + json.dumps(pytest_args + [f"--junitxml={worker_junit}"]) + "))", "", )), encoding="utf-8", ) - return runner + checker = directory / "run_execution.py" + checker.write_text( + "\n".join(( + "import pathlib, subprocess, sys", + f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", + f"source = pathlib.Path({json.dumps(str(worker_junit))})", + "if result.returncode != 0 or not source.is_file(): raise SystemExit(1)", + f"pathlib.Path({json.dumps(str(junit))}).write_bytes(source.read_bytes())", + "",)), encoding="utf-8", + ) + return checker, worker_junit def _run_test_node( @@ -732,11 +769,13 @@ def _run_test_node( home.mkdir(mode=0o700, parents=True) junit = temporary / "junit.xml" junit.touch(mode=0o600) - controller = _trusted_execution_runner(temporary, root, command[3:], junit) + controller, worker_junit = _trusted_execution_runner( + temporary, root, command[3:], junit + ) result, surviving = _managed_subprocess( [sys.executable, str(controller)], cwd=temporary, timeout=timeout_seconds, env=_pytest_environment(home), - writable_roots=(home.parent,), writable_files=(junit,), + writable_roots=(home.parent,), writable_files=(junit, worker_junit), readable_roots=(root,), ) if result.returncode == 124: @@ -782,7 +821,7 @@ def _collect_node_ids( "no:cacheprovider", path.as_posix(), ] - runner = _trusted_collection_runner( + runner, worker_output = _trusted_collection_runner( temporary, root, pytest_args, collection_output ) command = [sys.executable, str(runner)] @@ -801,7 +840,7 @@ def _collect_node_ids( result, surviving = _managed_subprocess( command, cwd=temporary, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), - writable_files=(collection_output,), + writable_files=(collection_output, worker_output), readable_roots=(root, _CHECKER_PYTEST_PROBE), ) if result.returncode == 124: diff --git a/pdd/sync_determine_operation.py b/pdd/sync_determine_operation.py index 8fcc162e5..122e6c953 100644 --- a/pdd/sync_determine_operation.py +++ b/pdd/sync_determine_operation.py @@ -1727,7 +1727,7 @@ def calculate_sha256(file_path: Path) -> Optional[str]: return None -def extract_include_deps(prompt_path: Path) -> Dict[str, str]: +def extract_include_deps(prompt_path: Path, *, version: int = 2) -> Dict[str, str]: """Extract include dependency paths and their hashes from a prompt file. Returns a dict mapping resolved dependency paths to their SHA256 hashes. @@ -1749,8 +1749,13 @@ def extract_include_deps(prompt_path: Path) -> Dict[str, str]: except OSError: return {} dependencies: Dict[str, str] = {} - for reference in parse_include_references(content): - declared = Path(reference.path) + declared_paths = ( + _legacy_include_references(content) + if version == 1 + else [reference.path for reference in parse_include_references(content)] + ) + for declared_text in declared_paths: + declared = Path(declared_text.strip()) candidates = ( (declared,) if declared.is_absolute() @@ -1964,7 +1969,7 @@ def calculate_current_hashes(paths: Dict[str, Any], stored_include_deps: Optiona # Issue #522: Hash prompt with dependencies hashes['prompt_hash'] = calculate_prompt_hash(file_path, stored_deps=stored_include_deps) # Also extract current include deps for persistence - hashes['include_deps'] = extract_include_deps(file_path) + hashes['include_deps'] = extract_include_deps(file_path, version=1) # If no deps found in prompt but we have stored deps, preserve them if not hashes['include_deps'] and stored_include_deps: # Re-hash stored deps to check for changes diff --git a/pdd/sync_main.py b/pdd/sync_main.py index 73a5c3573..008f4afe4 100644 --- a/pdd/sync_main.py +++ b/pdd/sync_main.py @@ -932,14 +932,10 @@ def sync_main( context_override=context_override, ) - from .continuous_sync import canonical_sync_enabled - - if canonical_sync_enabled(Path(pdd_files.get("prompt") or Path.cwd())): - raise RuntimeError( - "protected canonical sync blocks legacy production mutation; " - "use read-only reporting or trusted finalization until the " - "staged repair executor is enabled" - ) + from .sync_core.finalize import preflight_legacy_mutation + preflight_legacy_mutation( + {"prompt": Path(pdd_files.get("prompt") or Path.cwd())} + ) # Check fingerprint — skip if module is already fully synced _one_session_skipped = False diff --git a/pdd/update_main.py b/pdd/update_main.py index 1b1c704a6..42259658b 100644 --- a/pdd/update_main.py +++ b/pdd/update_main.py @@ -1249,6 +1249,22 @@ def update_main( :return: Tuple containing the updated prompt, total cost, and model name. """ quiet = ctx.obj.get("quiet", False) + from .sync_core.finalize import preflight_legacy_mutation + preflight_paths = None + if input_prompt_file or modified_code_file: + preflight_paths = { + "prompt": Path(input_prompt_file or modified_code_file), + "code": Path(modified_code_file) if modified_code_file else Path.cwd(), + } + try: + preflight_legacy_mutation(preflight_paths) + except Exception as exc: + from .sync_core.finalize import CanonicalFinalizationError + if isinstance(exc, CanonicalFinalizationError): + if not quiet: + rprint(f"[bold red]Error:[/bold red] {exc}") + raise click.exceptions.Exit(1) from exc + raise if repo: try: # Find the repo root by searching up from the current directory diff --git a/tests/test_sync_core_includes.py b/tests/test_sync_core_includes.py index a258de0da..7fe30078d 100644 --- a/tests/test_sync_core_includes.py +++ b/tests/test_sync_core_includes.py @@ -270,7 +270,7 @@ def test_manifest_output_alias_resolves_generated_code_relative_include(tmp_path def test_parent_prefixed_repository_path_is_rejected_even_with_same_tail(tmp_path) -> None: - prompt = tmp_path / "prompts/frontend/components/widget.prompt" + prompt = tmp_path / "prompts/widget.prompt" dependency = tmp_path / "frontend/src/context/AuthContext.tsx" prompt.parent.mkdir(parents=True) dependency.parent.mkdir(parents=True) @@ -278,7 +278,7 @@ def test_parent_prefixed_repository_path_is_rejected_even_with_same_tail(tmp_pat dependency.write_text("export const AuthContext = {};\n") with pytest.raises(IncludeGraphError, match="escapes repository"): build_include_closure( - PurePosixPath("prompts/frontend/components/widget.prompt"), + PurePosixPath("prompts/widget.prompt"), PathPolicy(tmp_path), ) diff --git a/tests/test_update_main.py b/tests/test_update_main.py index 0e3e32eeb..703d25134 100644 --- a/tests/test_update_main.py +++ b/tests/test_update_main.py @@ -53,7 +53,7 @@ def test_protected_update_cli_exits_nonzero_without_model_calls_or_writes( with patch("pdd.update_main.update_prompt") as model_call: result = CliRunner().invoke( - cli, ["--quiet", "update", str(prompt), str(modified), str(original)] + cli, ["update", str(prompt), str(modified), str(original)] ) assert result.exit_code != 0 From e5c63c5181b8d222f2c70e231c66c8b1de1416ad Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 17:38:59 -0700 Subject: [PATCH 073/128] fix(sync): preserve worker outcomes across checker boundary --- pdd/sync_core/runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 75ff69010..f3faa7815 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -677,6 +677,7 @@ def _trusted_collection_runner( """Create a process-separated checker and candidate collection worker.""" worker = directory / "collection_worker.py" worker_output = directory / "candidate-node-ids.json" + worker_output.touch(mode=0o600) worker.write_text( "\n".join( ( @@ -712,7 +713,7 @@ def _trusted_collection_runner( "import pathlib, subprocess, sys", f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", f"source = pathlib.Path({json.dumps(str(worker_output))})", - "if result.returncode != 0 or not source.is_file(): raise SystemExit(1)", + "if result.returncode not in (0, 5) or not source.stat().st_size: raise SystemExit(1)", f"pathlib.Path({json.dumps(str(collection_output))}).write_bytes(source.read_bytes())", "",)), encoding="utf-8", ) @@ -725,6 +726,7 @@ def _trusted_execution_runner( """Create a checker that never imports candidate code or pytest.""" worker = directory / "execution_worker.py" worker_junit = directory / "candidate-junit.xml" + worker_junit.touch(mode=0o600) worker.write_text( "\n".join(( "import os, sys", "import pytest", @@ -740,8 +742,9 @@ def _trusted_execution_runner( "import pathlib, subprocess, sys", f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", f"source = pathlib.Path({json.dumps(str(worker_junit))})", - "if result.returncode != 0 or not source.is_file(): raise SystemExit(1)", + "if not source.stat().st_size: raise SystemExit(1)", f"pathlib.Path({json.dumps(str(junit))}).write_bytes(source.read_bytes())", + "raise SystemExit(result.returncode)", "",)), encoding="utf-8", ) return checker, worker_junit From 8d64c7541e26e69ca03d08cc380f3d9c7ff0aef9 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 17:43:26 -0700 Subject: [PATCH 074/128] fix(sync): retain protected collection diagnostics --- pdd/sync_core/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index f3faa7815..73339f867 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -876,7 +876,7 @@ def _collect_node_ids( path.as_posix(), EvidenceOutcome.COLLECTION_ERROR, digest, - "trusted collection probe produced no valid node IDs: " + "protected collection probe produced no valid node IDs: " + (result.stderr or result.stdout)[-500:], ), (), From f8ea5fe9125d53625761a27ef37dd76ef2e5cb0a Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 19:14:37 -0700 Subject: [PATCH 075/128] test(sync): reproduce exhaustive review findings --- tests/test_sync_core_certificate.py | 24 ++++++++++ tests/test_sync_core_certificate_verifier.py | 9 ++++ tests/test_sync_core_identity_path_policy.py | 23 +++++++++ tests/test_sync_core_reporting.py | 29 ++++++++++++ tests/test_sync_core_runner.py | 50 ++++++++++++++++++++ tests/test_sync_core_transaction.py | 43 +++++++++++++++++ tests/test_sync_core_trust.py | 19 ++++++++ 7 files changed, 197 insertions(+) diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index cd577b03c..953f1eca1 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -27,9 +27,33 @@ _NightlyVerificationPolicy, _nightly_lineage, _nightly_streak, + _predicate, + RemoteCertificateSigner, ) +def test_global_predicate_cannot_downgrade_seven_night_minimum(monkeypatch) -> None: + monkeypatch.setattr("pdd.sync_core.certificate._scan_predicate", lambda *_a, **_k: True) + counts = {} + lifecycle = LifecycleResult(0, 0, 0, 0, 0, 0) + extra = {"nightly_streak": 1, "required_nightly_streak": 1} + assert _predicate(counts, lifecycle, extra, require_measurement=False) is False + + +def test_remote_certificate_signer_has_protected_timeout(monkeypatch) -> None: + def stalled(_command, **kwargs): + assert kwargs["timeout"] > 0 + raise subprocess.TimeoutExpired("protected-certificate-sign", kwargs["timeout"]) + + monkeypatch.setattr("pdd.sync_core.certificate.subprocess.run", stalled) + signer = RemoteCertificateSigner( + "trusted-ci", AttestationSigner("trusted-ci", b"r" * 32).public_key_bytes(), + ("protected-certificate-sign",), + ) + with pytest.raises(ValueError, match="timed out"): + signer.sign_bytes(b"certificate") + + CHECKER = CheckerIdentity( "c" * 64, "refs/tags/v-test-checker", diff --git a/tests/test_sync_core_certificate_verifier.py b/tests/test_sync_core_certificate_verifier.py index 4a2c5151b..3dec56b8b 100644 --- a/tests/test_sync_core_certificate_verifier.py +++ b/tests/test_sync_core_certificate_verifier.py @@ -78,3 +78,12 @@ def test_malformed_expectations_fail_closed(tmp_path, mutation) -> None: path.write_text(json.dumps(payload)) with pytest.raises(ValueError): load_expectations(path) + + +def test_protected_expectations_require_independent_seven_night_minimum(tmp_path) -> None: + payload = _payload() + payload["minimum_nightly_streak"] = 1 + path = tmp_path / "expectations.json" + path.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="seven"): + load_expectations(path) diff --git a/tests/test_sync_core_identity_path_policy.py b/tests/test_sync_core_identity_path_policy.py index 277307506..e3165d3a6 100644 --- a/tests/test_sync_core_identity_path_policy.py +++ b/tests/test_sync_core_identity_path_policy.py @@ -1,6 +1,8 @@ """Tests for stable repository identity and protected path handling.""" import os +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier from pathlib import PurePosixPath import pytest @@ -23,6 +25,27 @@ def test_repository_identity_is_initialized_once(tmp_path) -> None: assert initialize_repository_identity(tmp_path, "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa") == identity +def test_concurrent_repository_identity_initialization_returns_single_winner( + tmp_path, monkeypatch +) -> None: + barrier = Barrier(2) + real_replace = os.replace + + def racing_replace(source, destination): + barrier.wait(timeout=5) + real_replace(source, destination) + + monkeypatch.setattr("pdd.sync_core.identity.os.replace", racing_replace) + requested = ( + "3b4d7b1c-d6cc-4752-ba93-6b98d1a710e0", + "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa", + ) + with ThreadPoolExecutor(max_workers=2) as pool: + results = tuple(pool.map(lambda value: initialize_repository_identity(tmp_path, value), requested)) + persisted = read_repository_identity(tmp_path, require_persistent=True) + assert results == (persisted, persisted) + + def test_legacy_identity_is_report_only(tmp_path) -> None: first = read_repository_identity(tmp_path) second = read_repository_identity(tmp_path) diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index 18275a082..14a407392 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -336,6 +336,35 @@ def test_repository_identity_does_not_activate_enforcement_without_policy( assert canonical_sync_enabled(root) is False +def test_committed_policy_stays_active_when_worktree_policy_is_deleted(tmp_path) -> None: + root, _commit = _repository(tmp_path) + (root / ".pdd/sync-policy.json").unlink() + assert canonical_sync_enabled(root) is True + + +def test_scoped_canonical_compatibility_uses_selected_counts_and_qualified_names( + tmp_path, monkeypatch +) -> None: + canonical = { + "ok": False, + "counts": {"managed_units": 2, "trusted_in_sync": 1, "drifted": 0, + "unbaselined": 0, "corrupt": 0, "unknown": 0, + "conflict": 0, "failed": 0, "invalid": 0}, + "units": [{"subject": "prompts/commands/foo_python.prompt", + "baseline": "CURRENT", "semantic": "PASS", "in_sync": True, + "reason": "trusted", "changed_roles": []}], + } + monkeypatch.setattr( + "pdd.continuous_sync.build_canonical_report", lambda *_args, **_kwargs: canonical + ) + report = build_compatibility_report( + consumer="reconcile", root=tmp_path, modules=("commands/foo",) + ) + assert report["ok"] is True + assert report["summary"]["total"] == 1 + assert report["units"][0]["basename"] == "commands/foo" + + @pytest.mark.parametrize("protected_ref", ["missing-ref", "HEAD"]) def test_explicit_protected_mode_invalid_trust_input_fails_closed( tmp_path, monkeypatch, protected_ref diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index abcef7fe2..878228ec5 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -867,6 +867,56 @@ def test_candidate_controller_globals_cannot_forge_collection_pass(tmp_path: Pat assert executions[0].outcome is not EvidenceOutcome.PASS +def test_candidate_cannot_forge_worker_outputs_discovered_from_script_path( + tmp_path: Path, +) -> None: + content = ( + "import os, sys\nfrom pathlib import Path\n" + "directory = Path(sys.argv[0]).resolve().parent\n" + "if Path(sys.argv[0]).name == 'collection_worker.py':\n" + " (directory / 'candidate-node-ids.json').write_text(" + "'[\"tests/test_widget.py::test_widget\"]')\n" + " os._exit(0)\n" + "if Path(sys.argv[0]).name == 'execution_worker.py':\n" + " (directory / 'candidate-junit.xml').write_text(" + "'')\n" + " os._exit(0)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS + + +def test_runner_identity_binds_transitive_product_dependency(tmp_path: Path) -> None: + root, _commit = _repository( + tmp_path, "import product\ndef test_widget(): assert product.value == 1\n" + ) + (root / "product.py").write_text("from helper import value\n") + (root / "helper.py").write_text("value = 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "product closure") + before = _git(root, "rev-parse", "HEAD") + profile = _profile(root, before, (PurePosixPath("product.py"),)) + before_digest = runner_identity_digest(profile, root=root, ref=before) + (root / "helper.py").write_text("value = 2\n") + _git(root, "add", "helper.py") + _git(root, "commit", "-q", "-m", "change indirect helper") + after = _git(root, "rev-parse", "HEAD") + assert runner_identity_digest(profile, root=root, ref=after) != before_digest + + +def test_runner_identity_is_stable_across_interpreter_prefixes( + tmp_path: Path, monkeypatch +) -> None: + root, commit = _repository(tmp_path, "def test_widget(): assert True\n") + profile = _profile(root, commit) + monkeypatch.setattr("pdd.sync_core.runner.sys.executable", "/venv-a/bin/python") + first = runner_identity_digest(profile, root=root, ref=commit) + monkeypatch.setattr("pdd.sync_core.runner.sys.executable", "/venv-b/bin/python") + assert runner_identity_digest(profile, root=root, ref=commit) == first + + def test_pytest_plugin_guard_ignores_non_pytest_preview_prose(tmp_path: Path) -> None: root, _commit = _repository(tmp_path, "def test_widget(): assert True\n") (root / "setup.cfg").write_text( diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index faef84dce..45a70f557 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -63,6 +63,49 @@ def test_external_edit_after_prepare_is_conflict_without_writes(tmp_path) -> Non assert not (tmp_path / ".pdd/evidence/widget.json").exists() +def test_external_edit_after_first_install_prevents_commit(tmp_path) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-post-install-race", _writes()) + + def race(event): + if event == "after_install:0": + target.write_text("external = True\n") + + with pytest.raises(TransactionConflict, match="destination changed"): + manager.commit("tx-post-install-race", crash_hook=race) + assert target.read_text() == "external = True\n" + + +def test_rollback_preserves_external_edit_after_install(tmp_path, monkeypatch) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-rollback-race", _writes()) + original_install = manager._install_entry + calls = 0 + + def failing_install(transaction_dir, entry): + nonlocal calls + calls += 1 + if calls == 2: + raise TransactionError("later install failed") + original_install(transaction_dir, entry) + + monkeypatch.setattr(manager, "_install_entry", failing_install) + + def race(event): + if event == "after_install:0": + target.write_text("external = True\n") + + with pytest.raises(TransactionConflict, match="rollback conflict"): + manager.commit("tx-rollback-race", crash_hook=race) + assert target.read_text() == "external = True\n" + + @pytest.mark.parametrize("crash_event", ["after_committing", "after_install:0", "after_install:1"]) def test_process_death_recovers_committing_transaction(tmp_path, crash_event) -> None: (tmp_path / "src").mkdir() diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 817a9298d..b296d4252 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -100,6 +100,25 @@ def remote_sign(command, *, input, capture_output, check): assert evidence.attestation_id == request.attestation_id +def test_remote_attestation_signer_has_protected_timeout(monkeypatch) -> None: + def stalled(_command, **kwargs): + assert kwargs["timeout"] > 0 + raise subprocess.TimeoutExpired("protected-attestation-sign", kwargs["timeout"]) + + monkeypatch.setattr("pdd.sync_core.trust.subprocess.run", stalled) + signer = RemoteAttestationSigner( + SIGNER.issuer, SIGNER.public_key_bytes(), ("protected-attestation-sign",) + ) + with pytest.raises(AttestationError, match="timed out"): + signer.issue( + AttestationRequest( + "remote-attestation", _binding(), + (ObligationEvidence("test", EvidenceOutcome.PASS),), + "remote-nonce", NOW, + ) + ) + + def test_attestation_environment_forbids_local_private_key(monkeypatch) -> None: monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-secret") with pytest.raises(ValueError, match="local attestation signing keys are forbidden"): From 483cb80c186b2c438f2b00a90c46a13f81b62f2c Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 19:23:05 -0700 Subject: [PATCH 076/128] fix(sync): close exhaustive verification boundaries --- pdd/commands/sync_core.py | 2 +- pdd/continuous_sync.py | 40 +++++++++++++++------------ pdd/sync_core/certificate.py | 31 ++++++++++++++++----- pdd/sync_core/certificate_verifier.py | 6 ++++ pdd/sync_core/identity.py | 11 ++++++-- pdd/sync_core/runner.py | 8 ++++-- pdd/sync_core/signer_process.py | 32 +++++++++++++++++++++ pdd/sync_core/transaction.py | 26 +++++++++++++++-- pdd/sync_core/trust.py | 11 ++++---- tests/test_sync_core_certificate.py | 10 +++---- tests/test_sync_core_reporting.py | 1 + tests/test_sync_core_transaction.py | 2 +- tests/test_sync_core_trust.py | 10 +++---- 13 files changed, 141 insertions(+), 49 deletions(-) create mode 100644 pdd/sync_core/signer_process.py diff --git a/pdd/commands/sync_core.py b/pdd/commands/sync_core.py index ae13e6174..42c0f9746 100644 --- a/pdd/commands/sync_core.py +++ b/pdd/commands/sync_core.py @@ -152,7 +152,7 @@ def _load_nightly_observation(path: Path | None) -> NightlyObservation | None: @click.option("--merge-group", help="Exact protected PDD merge-group commit SHA.") @click.option("--full-inventory", is_flag=True) @click.option("--run-lifecycle-matrix", "run_matrix", is_flag=True) -@click.option("--require-nightly-streak", type=click.IntRange(min=1)) +@click.option("--require-nightly-streak", type=click.IntRange(min=7)) @click.option( "--nightly-ledger", type=click.Path(path_type=Path, dir_okay=False), diff --git a/pdd/continuous_sync.py b/pdd/continuous_sync.py index 23b9126e3..0b40f0584 100644 --- a/pdd/continuous_sync.py +++ b/pdd/continuous_sync.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from itertools import combinations -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Dict, Iterable, List, Optional from .operation_log import ( @@ -96,11 +96,6 @@ def canonical_sync_enabled(root: Path) -> bool: candidate = Path(root).resolve() if not candidate.is_dir(): candidate = candidate.parent - if protected_ref is None and not any( - (parent / ".pdd/sync-policy.json").is_file() - for parent in (candidate, *candidate.parents) - ): - return False root = repository_root(candidate) protected_ref = protected_ref or "HEAD" identity = subprocess.run( @@ -824,7 +819,9 @@ def _canonical_compatibility_report( classification = FAILURE_CLASSIFICATION projected.append( { - "basename": Path(unit["subject"]).stem.rsplit("_", 1)[0], + "basename": PurePosixPath(unit["subject"]).relative_to( + "prompts" + ).with_suffix("").as_posix().rsplit("_", 1)[0], "language": Path(unit["subject"]).stem.rsplit("_", 1)[-1], "classification": classification, "operation": "none", @@ -835,17 +832,26 @@ def _canonical_compatibility_report( } ) summary = _build_summary(projected) - counts = canonical["counts"] - summary["metadata_stale"] = counts["drifted"] - summary["conflicts"] = counts["conflict"] - summary["unbaselined"] = counts["unbaselined"] - summary["failures"] = ( - counts["corrupt"] + counts["unknown"] + counts["failed"] + counts["invalid"] - ) - summary["synced"] = counts["trusted_in_sync"] - summary["total"] = counts["managed_units"] + summary["metadata_stale"] = len([ + item for item in projected if item["classification"] in DRIFT_CLASSIFICATIONS + ]) + summary["conflicts"] = len([ + item for item in projected if item["classification"] == CONFLICT_CLASSIFICATION + ]) + summary["unbaselined"] = len([ + item for item in projected if item["classification"] == UNBASELINED_CLASSIFICATION + ]) + summary["failures"] = len([ + item for item in projected if item["classification"] == FAILURE_CLASSIFICATION + ]) + summary["synced"] = len([ + item for item in projected if item["classification"] == "IN_SYNC" + ]) + summary["total"] = len(projected) return { - "ok": canonical["ok"], + "ok": bool(projected) and all( + item["classification"] == "IN_SYNC" for item in projected + ), "consumer": consumer, "project_root": str(root), "summary": summary, diff --git a/pdd/sync_core/certificate.py b/pdd/sync_core/certificate.py index 157b2b523..b35b43485 100644 --- a/pdd/sync_core/certificate.py +++ b/pdd/sync_core/certificate.py @@ -25,6 +25,11 @@ CandidateArtifactProvenanceError, ) from .reporting import CanonicalReportOptions, build_canonical_report +from .signer_process import run_signer + + +MINIMUM_NIGHTLY_STREAK = 7 +SIGNER_TIMEOUT_SECONDS = 60 class CertificateSigner(Protocol): @@ -55,12 +60,12 @@ def public_key_bytes(self) -> bytes: def sign_bytes(self, payload: bytes) -> str: """Sign canonical bytes remotely and verify the response locally.""" - result = subprocess.run( - self._command, - input=payload, - capture_output=True, - check=False, - ) + try: + result = run_signer( + self._command, payload, timeout=SIGNER_TIMEOUT_SECONDS + ) + except subprocess.TimeoutExpired as exc: + raise ValueError("remote certificate signer timed out") from exc if result.returncode != 0: raise ValueError("remote certificate signer rejected the request") try: @@ -684,7 +689,8 @@ def _predicate( ) -> bool: return _scan_predicate(counts, lifecycle, extra, require_measurement=require_measurement) and ( - extra["nightly_streak"] >= extra["required_nightly_streak"] + extra["nightly_streak"] >= MINIMUM_NIGHTLY_STREAK + and extra["required_nightly_streak"] >= MINIMUM_NIGHTLY_STREAK ) @@ -944,6 +950,8 @@ def _build_global_certificate_from_targets( """Build and sign the complete cross-repository machine predicate.""" if not targets: raise ValueError("global certificate requires at least one repository") + if options.required_nightly_streak < MINIMUM_NIGHTLY_STREAK: + raise ValueError("global certificate requires at least seven nightly observations") if options.checker_identity is None: raise ValueError("global certificate requires released checker identity") if options.candidate_artifact_policy is None: @@ -1123,6 +1131,7 @@ def verify_global_certificate( expected_repository_ids: dict[str, str], expected_checker_identity: CheckerIdentity, expected_candidate_artifact_policy: CandidateArtifactPolicy, + expected_minimum_nightly_streak: int = MINIMUM_NIGHTLY_STREAK, now: datetime | None = None, maximum_age: timedelta = timedelta(minutes=15), ) -> bool: @@ -1135,6 +1144,14 @@ def verify_global_certificate( return False if certificate.get("ok") is not True or certificate.get("scan_ok") is not True: return False + counts = certificate.get("counts") + if ( + expected_minimum_nightly_streak < MINIMUM_NIGHTLY_STREAK + or not isinstance(counts, dict) + or counts.get("nightly_streak", 0) < expected_minimum_nightly_streak + or counts.get("required_nightly_streak", 0) < expected_minimum_nightly_streak + ): + return False if certificate.get("checker") != expected_checker_identity.payload(): return False if ( diff --git a/pdd/sync_core/certificate_verifier.py b/pdd/sync_core/certificate_verifier.py index 61e937b03..5dc4a64f4 100644 --- a/pdd/sync_core/certificate_verifier.py +++ b/pdd/sync_core/certificate_verifier.py @@ -23,6 +23,7 @@ class CertificateExpectations: candidate_artifact_policy: CandidateArtifactPolicy repository_shas: dict[str, tuple[str, str]] repository_ids: dict[str, str] + minimum_nightly_streak: int = 7 def _sha(value: Any) -> str: @@ -57,6 +58,9 @@ def load_expectations(path: Path) -> CertificateExpectations: payload = json.loads(Path(path).read_text(encoding="utf-8")) if not isinstance(payload, dict) or payload.get("schema_version") != 1: raise ValueError("expectations schema is invalid") + minimum_nightly_streak = int(payload.get("minimum_nightly_streak", 7)) + if minimum_nightly_streak < 7: + raise ValueError("protected expectations require seven nightly observations") checker_payload = payload["checker"] candidate_policy_payload = payload["candidate_artifact_policy"] repositories = payload["repositories"] @@ -92,6 +96,7 @@ def load_expectations(path: Path) -> CertificateExpectations: candidate_policy, repository_shas, repository_ids, + minimum_nightly_streak, ) @@ -114,6 +119,7 @@ def main() -> None: expected_repository_ids=expected.repository_ids, expected_checker_identity=expected.checker, expected_candidate_artifact_policy=expected.candidate_artifact_policy, + expected_minimum_nightly_streak=expected.minimum_nightly_streak, ) except (OSError, ValueError, json.JSONDecodeError): verified = False diff --git a/pdd/sync_core/identity.py b/pdd/sync_core/identity.py index c4295b05c..5ce390c05 100644 --- a/pdd/sync_core/identity.py +++ b/pdd/sync_core/identity.py @@ -82,14 +82,21 @@ def initialize_repository_identity( if state_dir.exists() and (state_dir.is_symlink() or not state_dir.is_dir()): raise RepositoryIdentityError(".pdd must be a real directory") state_dir.mkdir(mode=0o700, parents=True, exist_ok=True) - temporary = state_dir / f".{identity_path.name}.{os.getpid()}.tmp" + temporary = state_dir / ( + f".{identity_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + ) descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: with os.fdopen(descriptor, "w", encoding="ascii") as handle: handle.write(value + "\n") handle.flush() os.fsync(handle.fileno()) - os.replace(temporary, identity_path) + try: + os.link(temporary, identity_path) + except FileExistsError: + temporary.unlink(missing_ok=True) + return read_repository_identity(repository_root, require_persistent=True) + temporary.unlink() fsync_directory(state_dir) except BaseException: temporary.unlink(missing_ok=True) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 73339f867..4b3d0a3c8 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -457,6 +457,10 @@ def _support_digest( for path in obligation.code_under_test_paths ) closure = _pytest_support_closure(root, ref, tests, product) + product_closure = _transitive_support_blobs( + root, ref, pending=product, included=set(product) + ) + closure = tuple(sorted(set(closure + product_closure))) digest = hashlib.sha256() for path, content in closure: digest.update(path.as_posix().encode() + b"\0" + content + b"\0") @@ -518,7 +522,7 @@ def runner_identity_digest( payload = { "tool_version": TRUSTED_RUNNER_VERSION, "pytest_command": [ - sys.executable, + "", "-m", "pytest", "-q", @@ -527,7 +531,7 @@ def runner_identity_digest( "--junitxml=", ], "pytest_collection_command": [ - sys.executable, + "", "-m", "pytest", "--collect-only", diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py new file mode 100644 index 000000000..1469052f9 --- /dev/null +++ b/pdd/sync_core/signer_process.py @@ -0,0 +1,32 @@ +"""Bounded process-tree execution for protected signer adapters.""" + +from __future__ import annotations + +import os +import signal +import subprocess + + +def run_signer( + command: tuple[str, ...], payload: bytes, *, timeout: float +) -> subprocess.CompletedProcess[bytes]: + """Run a signer in a new process group and reap the complete group on timeout.""" + process = subprocess.Popen( # pylint: disable=consider-using-with + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate(payload, timeout=timeout) + except subprocess.TimeoutExpired as exc: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired( + command, timeout, output=stdout, stderr=stderr + ) from exc + return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) diff --git a/pdd/sync_core/transaction.py b/pdd/sync_core/transaction.py index 6a48d63f6..8092a401e 100644 --- a/pdd/sync_core/transaction.py +++ b/pdd/sync_core/transaction.py @@ -499,6 +499,11 @@ def _restore_entries( if not isinstance(precondition, dict): raise TransactionError("transaction rollback precondition is malformed") before = _parse_state(precondition) + desired = FileState( + True, str(item["desired_digest"]), str(item["desired_mode"]), "regular" + ) + if self._destination_state(relpath) != desired: + raise TransactionConflict(f"rollback conflict: {relpath}") rollback_name = item.get("rollback_blob") if not before.exists: self._unlink_destination(relpath) @@ -522,6 +527,7 @@ def commit( *, crash_hook: Optional[Callable[[str], None]] = None, ) -> TransactionResult: + # pylint: disable=too-many-locals,too-many-branches """Install all prepared writes after a durable COMMITTING marker.""" transaction_dir = self._transaction_dir(transaction_id) payload = self._read_journal(transaction_dir) @@ -557,11 +563,25 @@ def commit( changed.append(PurePosixPath(str(item["relpath"]))) self._write_journal(transaction_dir, payload) hook(f"after_install:{index}") - except TransactionError: - self._restore_entries(transaction_dir, entries) + for item in entries: + if not isinstance(item, dict): + raise TransactionError("transaction entry is malformed") + relpath = PurePosixPath(str(item["relpath"])) + desired = FileState( + True, str(item["desired_digest"]), + str(item["desired_mode"]), "regular", + ) + if self._destination_state(relpath) != desired: + raise TransactionConflict(f"destination changed: {relpath}") + except TransactionError as exc: + try: + self._restore_entries(transaction_dir, entries) + except TransactionConflict: + self._write_journal(transaction_dir, payload) + raise payload["phase"] = TransactionPhase.ROLLED_BACK.value self._write_journal(transaction_dir, payload) - raise + raise exc payload["phase"] = TransactionPhase.COMMITTED.value self._write_journal(transaction_dir, payload) hook("after_commit") diff --git a/pdd/sync_core/trust.py b/pdd/sync_core/trust.py index d3ba203ea..86f3fc13b 100644 --- a/pdd/sync_core/trust.py +++ b/pdd/sync_core/trust.py @@ -18,6 +18,7 @@ Ed25519PrivateKey, Ed25519PublicKey, ) +from .signer_process import run_signer from .descriptor_store import DescriptorStoreError, update_json from .durability import fsync_directory from .types import EvidenceOutcome, ObligationEvidence, UnitId @@ -361,12 +362,10 @@ def public_key_bytes(self) -> bytes: def issue(self, request: AttestationRequest) -> AttestationEnvelope: """Send canonical claims to the remote authority and verify its signature.""" envelope = _unsigned_envelope(self.issuer, request) - result = subprocess.run( - self._command, - input=envelope.payload(), - capture_output=True, - check=False, - ) + try: + result = run_signer(self._command, envelope.payload(), timeout=60) + except subprocess.TimeoutExpired as exc: + raise AttestationError("remote attestation signer timed out") from exc if result.returncode != 0: raise AttestationError("remote attestation signer rejected the request") try: diff --git a/tests/test_sync_core_certificate.py b/tests/test_sync_core_certificate.py index 953f1eca1..55a66c440 100644 --- a/tests/test_sync_core_certificate.py +++ b/tests/test_sync_core_certificate.py @@ -41,11 +41,11 @@ def test_global_predicate_cannot_downgrade_seven_night_minimum(monkeypatch) -> N def test_remote_certificate_signer_has_protected_timeout(monkeypatch) -> None: - def stalled(_command, **kwargs): + def stalled(_command, _payload, **kwargs): assert kwargs["timeout"] > 0 raise subprocess.TimeoutExpired("protected-certificate-sign", kwargs["timeout"]) - monkeypatch.setattr("pdd.sync_core.certificate.subprocess.run", stalled) + monkeypatch.setattr("pdd.sync_core.certificate.run_signer", stalled) signer = RemoteCertificateSigner( "trusted-ci", AttestationSigner("trusted-ci", b"r" * 32).public_key_bytes(), ("protected-certificate-sign",), @@ -293,13 +293,13 @@ def test_certificate_signing_uses_remote_verified_authority( "PDD_CERTIFICATE_SIGNER_COMMAND", json.dumps(["protected-kms-sign"]) ) - def remote_sign(command, *, input, capture_output, check): + def remote_sign(command, input, *, timeout): assert command == ("protected-kms-sign",) - assert capture_output is True and check is False + assert timeout > 0 signature = authority.sign_bytes(input).encode("ascii") return subprocess.CompletedProcess(command, 0, stdout=signature, stderr=b"") - monkeypatch.setattr("pdd.sync_core.certificate.subprocess.run", remote_sign) + monkeypatch.setattr("pdd.sync_core.certificate.run_signer", remote_sign) signer = signer_from_environment() assert signer.sign_bytes(b"canonical certificate") == authority.sign_bytes( b"canonical certificate" diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index 14a407392..c18c6c77b 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -357,6 +357,7 @@ def test_scoped_canonical_compatibility_uses_selected_counts_and_qualified_names monkeypatch.setattr( "pdd.continuous_sync.build_canonical_report", lambda *_args, **_kwargs: canonical ) + monkeypatch.setattr("pdd.continuous_sync.canonical_sync_enabled", lambda _root: True) report = build_compatibility_report( consumer="reconcile", root=tmp_path, modules=("commands/foo",) ) diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index 45a70f557..3b4a36b7f 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -74,7 +74,7 @@ def race(event): if event == "after_install:0": target.write_text("external = True\n") - with pytest.raises(TransactionConflict, match="destination changed"): + with pytest.raises(TransactionConflict, match="rollback conflict"): manager.commit("tx-post-install-race", crash_hook=race) assert target.read_text() == "external = True\n" diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index b296d4252..29f986df5 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -79,9 +79,9 @@ def test_remote_attestation_authority_signature_is_verified(monkeypatch) -> None NOW, ) - def remote_sign(command, *, input, capture_output, check): + def remote_sign(command, input, *, timeout): assert command == ("protected-attestation-sign",) - assert capture_output is True and check is False + assert timeout > 0 return subprocess.CompletedProcess( command, 0, @@ -89,7 +89,7 @@ def remote_sign(command, *, input, capture_output, check): stderr=b"", ) - monkeypatch.setattr("pdd.sync_core.trust.subprocess.run", remote_sign) + monkeypatch.setattr("pdd.sync_core.trust.run_signer", remote_sign) signer = RemoteAttestationSigner( SIGNER.issuer, SIGNER.public_key_bytes(), ("protected-attestation-sign",) ) @@ -101,11 +101,11 @@ def remote_sign(command, *, input, capture_output, check): def test_remote_attestation_signer_has_protected_timeout(monkeypatch) -> None: - def stalled(_command, **kwargs): + def stalled(_command, _payload, **kwargs): assert kwargs["timeout"] > 0 raise subprocess.TimeoutExpired("protected-attestation-sign", kwargs["timeout"]) - monkeypatch.setattr("pdd.sync_core.trust.subprocess.run", stalled) + monkeypatch.setattr("pdd.sync_core.trust.run_signer", stalled) signer = RemoteAttestationSigner( SIGNER.issuer, SIGNER.public_key_bytes(), ("protected-attestation-sign",) ) From e0f0b820eac92611226d57e5450849f428448e19 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 19:29:36 -0700 Subject: [PATCH 077/128] fix(sync): separate candidate result channels --- pdd/sync_core/runner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 4b3d0a3c8..92b9c84af 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -8,6 +8,7 @@ import ast import configparser import json +import os import shlex import subprocess import sys @@ -464,7 +465,9 @@ def _support_digest( digest = hashlib.sha256() for path, content in closure: digest.update(path.as_posix().encode() + b"\0" + content + b"\0") - return digest.hexdigest(), tuple(path for path, _content in closure) + return digest.hexdigest(), tuple( + path for path, _content in closure if path not in set(product) + ) def _config_loads_plugin(root: Path, ref: str) -> bool: @@ -680,7 +683,7 @@ def _trusted_collection_runner( ) -> tuple[Path, Path]: """Create a process-separated checker and candidate collection worker.""" worker = directory / "collection_worker.py" - worker_output = directory / "candidate-node-ids.json" + worker_output = directory / f"collection-{os.urandom(16).hex()}.json" worker_output.touch(mode=0o600) worker.write_text( "\n".join( @@ -729,7 +732,7 @@ def _trusted_execution_runner( ) -> tuple[Path, Path]: """Create a checker that never imports candidate code or pytest.""" worker = directory / "execution_worker.py" - worker_junit = directory / "candidate-junit.xml" + worker_junit = directory / f"execution-{os.urandom(16).hex()}.xml" worker_junit.touch(mode=0o600) worker.write_text( "\n".join(( From 9c99a08d74e50f92c7afbe99146a321d166920f6 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 20:06:54 -0700 Subject: [PATCH 078/128] fix(sync): preserve legacy policy fast path --- pdd/continuous_sync.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pdd/continuous_sync.py b/pdd/continuous_sync.py index 0b40f0584..b8724c43e 100644 --- a/pdd/continuous_sync.py +++ b/pdd/continuous_sync.py @@ -96,6 +96,24 @@ def canonical_sync_enabled(root: Path) -> bool: candidate = Path(root).resolve() if not candidate.is_dir(): candidate = candidate.parent + if protected_ref is None: + ancestors = (candidate, *candidate.parents) + has_worktree_policy = any( + (parent / ".pdd/sync-policy.json").is_file() for parent in ancestors + ) + repository_marker = next( + ( + parent / ".git" + for parent in ancestors + if (parent / ".git").exists() + ), + None, + ) + has_repository_directory = bool( + repository_marker is not None and repository_marker.is_dir() + ) + if not has_worktree_policy and not has_repository_directory: + return False root = repository_root(candidate) protected_ref = protected_ref or "HEAD" identity = subprocess.run( From 81b65524c813b57148b6a0360a6147755864353f Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 20:19:56 -0700 Subject: [PATCH 079/128] test(sync): bound canonical policy activation --- tests/test_sync_core_reporting.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index c18c6c77b..b2ec1e264 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -342,6 +342,16 @@ def test_committed_policy_stays_active_when_worktree_policy_is_deleted(tmp_path) assert canonical_sync_enabled(root) is True +def test_policy_free_linked_worktree_avoids_git_subprocess(tmp_path, monkeypatch) -> None: + (tmp_path / ".git").write_text("gitdir: /protected/main/.git/worktrees/unit\n") + + def unexpected_subprocess(*_args, **_kwargs): + raise AssertionError("legacy worktree must not enter protected Git lookup") + + monkeypatch.setattr("pdd.continuous_sync.subprocess.run", unexpected_subprocess) + assert canonical_sync_enabled(tmp_path) is False + + def test_scoped_canonical_compatibility_uses_selected_counts_and_qualified_names( tmp_path, monkeypatch ) -> None: From c219f97b008d0ef03e7d9f6c6cf2c3f852ce9646 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 20:32:06 -0700 Subject: [PATCH 080/128] ci: bound broad unit suite with measured margin --- .github/workflows/unit-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b20c0eb6d..ed283ba28 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -29,7 +29,8 @@ jobs: if: github.event.pull_request.draft != true name: Run Unit Tests runs-on: ubuntu-latest - timeout-minutes: 30 + # The measured green broad suite takes ~30 minutes; retain bounded 50% headroom. + timeout-minutes: 45 env: PDD_PATH: ${{ github.workspace }}/pdd From d6963ba0ef1a4fdb2ca66520969bec978975559d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sat, 11 Jul 2026 23:47:05 -0700 Subject: [PATCH 081/128] fix(sync): cache committed policy activation --- pdd/continuous_sync.py | 122 +++++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/pdd/continuous_sync.py b/pdd/continuous_sync.py index b8724c43e..627d64cc4 100644 --- a/pdd/continuous_sync.py +++ b/pdd/continuous_sync.py @@ -6,6 +6,7 @@ import subprocess from dataclasses import dataclass from datetime import datetime, timezone +from functools import lru_cache from itertools import combinations from pathlib import Path, PurePosixPath from typing import Any, Dict, Iterable, List, Optional @@ -90,6 +91,90 @@ def repository_root(start: Optional[Path] = None) -> Path: return Path(result.stdout.strip()).resolve() +def _git_root_from_marker(start: Path) -> Optional[Path]: + """Resolve the enclosing worktree root without spawning Git.""" + candidate = start if start.is_dir() else start.parent + for parent in (candidate, *candidate.parents): + if (parent / ".git").exists(): + return parent + return None + + +def _git_head_token(root: Path) -> str: + """Return a filesystem token that changes when checked-out HEAD changes.""" + marker = root / ".git" + git_dir = marker + if marker.is_file(): + value = marker.read_text(encoding="utf-8").strip() + if not value.startswith("gitdir:"): + return value + git_dir = Path(value.split(":", 1)[1].strip()) + if not git_dir.is_absolute(): + git_dir = (root / git_dir).resolve() + try: + value = (git_dir / "HEAD").read_text(encoding="utf-8").strip() + except OSError: + return "" + if value.startswith("ref:"): + ref_name = value.split(":", 1)[1].strip() + common_dir = git_dir + try: + common_value = (git_dir / "commondir").read_text(encoding="utf-8").strip() + common_dir = Path(common_value) + if not common_dir.is_absolute(): + common_dir = (git_dir / common_dir).resolve() + except OSError: + pass + ref = common_dir / ref_name + try: + return f"{value}:{ref.read_text(encoding='utf-8').strip()}" + except OSError: + try: + packed = (common_dir / "packed-refs").read_text(encoding="utf-8") + except OSError: + packed = "" + suffix = f" {ref_name}" + for line in packed.splitlines(): + if line.endswith(suffix): + return f"{value}:{line.split(' ', 1)[0]}" + return value + + +@lru_cache(maxsize=128) +def _committed_canonical_policy( + root_value: str, + protected_ref: str, + head_token: str, +) -> tuple[bool, str]: + """Read activation once per repository, protected ref, and checked-out HEAD.""" + del head_token # Included solely to invalidate the cache after HEAD movement. + root = Path(root_value) + policy = subprocess.run( + ["git", "show", f"{protected_ref}:.pdd/sync-policy.json"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if policy.returncode != 0: + return False, "policy cannot be resolved" + identity = subprocess.run( + ["git", "cat-file", "-e", f"{protected_ref}:.pdd/repository-id"], + cwd=root, + capture_output=True, + check=False, + ) + if identity.returncode != 0: + return False, "identity cannot be resolved" + try: + payload = json.loads(policy.stdout) + except json.JSONDecodeError: + return False, "policy is malformed" + if payload != {"schema_version": 1, "enforcement": "active"}: + return False, "policy is not active" + return True, "" + + def canonical_sync_enabled(root: Path) -> bool: """Return whether protected committed policy activates canonical sync.""" protected_ref = os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") @@ -114,38 +199,17 @@ def canonical_sync_enabled(root: Path) -> bool: ) if not has_worktree_policy and not has_repository_directory: return False - root = repository_root(candidate) - protected_ref = protected_ref or "HEAD" - identity = subprocess.run( - ["git", "cat-file", "-e", f"{protected_ref}:.pdd/repository-id"], - cwd=root, - capture_output=True, - check=False, - ) - if identity.returncode != 0: - if os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: - raise ValueError("explicit protected sync identity cannot be resolved") + root = _git_root_from_marker(candidate) + if root is None: + if protected_ref is not None: + raise ValueError("explicit protected sync repository cannot be resolved") return False - policy = subprocess.run( - ["git", "show", f"{protected_ref}:.pdd/sync-policy.json"], - cwd=root, - capture_output=True, - text=True, - check=False, + protected_ref = protected_ref or "HEAD" + active, reason = _committed_canonical_policy( + str(root), protected_ref, _git_head_token(root) ) - if policy.returncode != 0: - if os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: - raise ValueError("explicit protected sync policy cannot be resolved") - return False - try: - payload = json.loads(policy.stdout) - except json.JSONDecodeError as exc: - if os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: - raise ValueError("explicit protected sync policy is malformed") from exc - return False - active = payload == {"schema_version": 1, "enforcement": "active"} if not active and os.environ.get("PDD_SYNC_PROTECTED_BASE_SHA") is not None: - raise ValueError("explicit protected sync policy is not active") + raise ValueError(f"explicit protected sync {reason}") return active From 1b08d03c92c5cae276be065ad0a05896d2759baa Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 00:32:10 -0700 Subject: [PATCH 082/128] test(sync): reproduce high review boundary gaps --- tests/test_sync_core_reporting.py | 46 +++++++++++++++++++++++++++++ tests/test_sync_core_runner.py | 33 +++++++++++++++++++-- tests/test_sync_core_transaction.py | 30 +++++++++++++++++++ tests/test_sync_core_trust.py | 15 ++++++++++ tests/test_sync_orchestration.py | 17 +++++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) diff --git a/tests/test_sync_core_reporting.py b/tests/test_sync_core_reporting.py index b2ec1e264..204c65438 100644 --- a/tests/test_sync_core_reporting.py +++ b/tests/test_sync_core_reporting.py @@ -342,6 +342,19 @@ def test_committed_policy_stays_active_when_worktree_policy_is_deleted(tmp_path) assert canonical_sync_enabled(root) is True +@pytest.mark.parametrize("mutation", ["delete", "rename"]) +def test_linked_worktree_committed_policy_stays_active(tmp_path, mutation) -> None: + root, _commit = _repository(tmp_path) + linked = tmp_path / "linked" + _git(root, "worktree", "add", "-q", str(linked), "-b", f"linked-{mutation}") + policy = linked / ".pdd/sync-policy.json" + if mutation == "delete": + policy.unlink() + else: + policy.rename(policy.with_suffix(".disabled")) + assert canonical_sync_enabled(linked) is True + + def test_policy_free_linked_worktree_avoids_git_subprocess(tmp_path, monkeypatch) -> None: (tmp_path / ".git").write_text("gitdir: /protected/main/.git/worktrees/unit\n") @@ -376,6 +389,39 @@ def test_scoped_canonical_compatibility_uses_selected_counts_and_qualified_names assert report["units"][0]["basename"] == "commands/foo" +def test_real_manifest_path_qualified_module_selects_one_duplicate_leaf(tmp_path) -> None: + root, _commit = _repository(tmp_path) + for scope in ("commands", "jobs"): + (root / "prompts" / scope).mkdir() + (root / "src" / scope).mkdir() + (root / "prompts" / scope / "foo_python.prompt").write_text( + f"REQ-{scope}: Build {scope} foo\n" + ) + (root / "src" / scope / "foo.py").write_text("value = 1\n") + architecture = json.loads((root / "architecture.json").read_text()) + architecture.extend([ + {"filename": "commands/foo_python.prompt", "filepath": "src/commands/foo.py"}, + {"filename": "jobs/foo_python.prompt", "filepath": "src/jobs/foo.py"}, + ]) + (root / "architecture.json").write_text(json.dumps(architecture)) + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "duplicate qualified leaves") + head = _git(root, "rev-parse", "HEAD") + options = _options(tmp_path, head) + report = build_canonical_report( + root, CanonicalReportOptions( + base_ref=options.base_ref, + head_ref=options.head_ref, + modules=("commands/foo",), + replay_ledger_path=options.replay_ledger_path, + now=options.now, + ) + ) + assert [item["subject"] for item in report["units"]] == [ + "prompts/commands/foo_python.prompt" + ] + + @pytest.mark.parametrize("protected_ref", ["missing-ref", "HEAD"]) def test_explicit_protected_mode_invalid_trust_input_fails_closed( tmp_path, monkeypatch, protected_ref diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 878228ec5..fe4fab59c 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -874,11 +874,11 @@ def test_candidate_cannot_forge_worker_outputs_discovered_from_script_path( "import os, sys\nfrom pathlib import Path\n" "directory = Path(sys.argv[0]).resolve().parent\n" "if Path(sys.argv[0]).name == 'collection_worker.py':\n" - " (directory / 'candidate-node-ids.json').write_text(" + " next(directory.glob('collection-*.json')).write_text(" "'[\"tests/test_widget.py::test_widget\"]')\n" " os._exit(0)\n" "if Path(sys.argv[0]).name == 'execution_worker.py':\n" - " (directory / 'candidate-junit.xml').write_text(" + " next(directory.glob('execution-*.xml')).write_text(" "'')\n" " os._exit(0)\n" "def test_widget(): assert False\n" @@ -888,6 +888,35 @@ def test_candidate_cannot_forge_worker_outputs_discovered_from_script_path( assert executions[0].outcome is not EvidenceOutcome.PASS +def test_runner_identity_fails_closed_for_dynamic_product_loader(tmp_path: Path) -> None: + root, _commit = _repository( + tmp_path, "import product\ndef test_widget(): assert product.value == 1\n" + ) + (root / "product.py").write_text( + "import importlib\nvalue = importlib.import_module('helper').value\n" + ) + (root / "helper.py").write_text("value = 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "dynamic product closure") + commit = _git(root, "rev-parse", "HEAD") + profile = _profile(root, commit, (PurePosixPath("product.py"),)) + with pytest.raises(ValueError, match="dynamic product dependency"): + runner_identity_digest(profile, root=root, ref=commit) + + +def test_runner_identity_binds_measured_runtime_and_checker( + tmp_path: Path, monkeypatch +) -> None: + root, commit = _repository(tmp_path, "def test_widget(): assert True\n") + profile = _profile(root, commit) + first = runner_identity_digest(profile, root=root, ref=commit) + monkeypatch.setattr("pdd.sync_core.runner.platform.python_version", lambda: "99.1") + assert runner_identity_digest(profile, root=root, ref=commit) != first + monkeypatch.undo() + monkeypatch.setattr("pdd.sync_core.runner._checker_artifact_digest", lambda: "0" * 64) + assert runner_identity_digest(profile, root=root, ref=commit) != first + + def test_runner_identity_binds_transitive_product_dependency(tmp_path: Path) -> None: root, _commit = _repository( tmp_path, "import product\ndef test_widget(): assert product.value == 1\n" diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index 3b4a36b7f..8c86ba50f 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -128,6 +128,36 @@ def crash(event): assert manager.recover("tx-crash").no_op is True +def test_recovery_race_cannot_false_commit(tmp_path, monkeypatch) -> None: + (tmp_path / "src").mkdir() + target = tmp_path / "src/widget.py" + target.write_text("value = 1\n") + manager = TransactionManager(tmp_path) + manager.prepare("tx-recovery-race", _writes()) + + def crash(event): + if event == "after_committing": + raise SystemExit + + with pytest.raises(SystemExit): + manager.commit("tx-recovery-race", crash_hook=crash) + original = manager._install_entry + calls = 0 + + def raced_install(transaction_dir, entry): + nonlocal calls + original(transaction_dir, entry) + calls += 1 + if calls == 1: + target.write_text("external = True\n") + + monkeypatch.setattr(manager, "_install_entry", raced_install) + with pytest.raises(TransactionConflict, match="destination changed"): + manager.recover("tx-recovery-race") + assert target.read_text() == "external = True\n" + assert manager.incomplete() == ("tx-recovery-race",) + + def test_prepared_transaction_recovers_by_rollback_without_destination_write(tmp_path) -> None: (tmp_path / "src").mkdir() target = tmp_path / "src/widget.py" diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 29f986df5..73306fbc0 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -3,6 +3,8 @@ import base64 import json import subprocess +import sys +import time from dataclasses import replace from datetime import datetime, timedelta, timezone from pathlib import PurePosixPath @@ -22,6 +24,7 @@ ) from pdd.sync_core.trust import ValidationEvidence from pdd.sync_core.finalize import attestation_signer_from_environment +from pdd.sync_core.signer_process import run_signer from pdd.sync_core.types import ObligationEvidence @@ -119,6 +122,18 @@ def stalled(_command, _payload, **kwargs): ) +def test_signer_timeout_is_bounded_with_detached_pipe_holder() -> None: + script = ( + "import subprocess,sys,time; " + "subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], " + "start_new_session=True); time.sleep(30)" + ) + started = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired): + run_signer((sys.executable, "-c", script), b"", timeout=0.1) + assert time.monotonic() - started < 1.5 + + def test_attestation_environment_forbids_local_private_key(monkeypatch) -> None: monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-secret") with pytest.raises(ValueError, match="local attestation signing keys are forbidden"): diff --git a/tests/test_sync_orchestration.py b/tests/test_sync_orchestration.py index 1b841fec0..45d64b2b3 100644 --- a/tests/test_sync_orchestration.py +++ b/tests/test_sync_orchestration.py @@ -3119,6 +3119,23 @@ def test_protected_canonical_mode_blocks_legacy_generator_before_write( orchestration_fixture["code_generator_main"].assert_not_called() +def test_protected_sync_preflights_before_log_or_lock(orchestration_fixture): + orchestration_fixture["sync_determine_operation"].side_effect = [ + SyncDecision(operation="generate", reason="prompt changed") + ] + with patch( + "pdd.sync_core.finalize.preflight_legacy_mutation", + side_effect=RuntimeError("protected preflight"), + ) as preflight, patch("pdd.sync_orchestration.log_event") as log_event: + result = sync_orchestration( + basename="calculator", language="python", quiet=True, budget=1.0 + ) + preflight.assert_called_once() + log_event.assert_not_called() + orchestration_fixture["SyncLock"].assert_not_called() + assert result["success"] is False + + class TestCoverageTargetSelection: """Regression tests for selecting the correct `--cov` target.""" From 064b41e7bba37970f87a0c38a776f52f1586d6b5 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 00:38:04 -0700 Subject: [PATCH 083/128] fix(sync): close high review verification boundaries --- pdd/continuous_sync.py | 26 +++++++++-- pdd/sync_core/reporting.py | 9 +++- pdd/sync_core/runner.py | 37 ++++++++++++++-- pdd/sync_core/signer_process.py | 67 ++++++++++++++++++++++++++++- pdd/sync_core/transaction.py | 20 +++++++-- pdd/sync_orchestration.py | 11 +++++ tests/test_sync_core_transaction.py | 2 +- 7 files changed, 159 insertions(+), 13 deletions(-) diff --git a/pdd/continuous_sync.py b/pdd/continuous_sync.py index 627d64cc4..82e7c9558 100644 --- a/pdd/continuous_sync.py +++ b/pdd/continuous_sync.py @@ -194,10 +194,14 @@ def canonical_sync_enabled(root: Path) -> bool: ), None, ) - has_repository_directory = bool( - repository_marker is not None and repository_marker.is_dir() + has_repository_marker = bool( + repository_marker is not None + and ( + repository_marker.is_dir() + or _valid_gitdir_file(repository_marker) + ) ) - if not has_worktree_policy and not has_repository_directory: + if not has_worktree_policy and not has_repository_marker: return False root = _git_root_from_marker(candidate) if root is None: @@ -213,6 +217,22 @@ def canonical_sync_enabled(root: Path) -> bool: return active +def _valid_gitdir_file(marker: Path) -> bool: + """Return whether a file-form linked-worktree marker names a live gitdir.""" + if not marker.is_file(): + return False + try: + prefix, value = marker.read_text(encoding="utf-8").strip().split(":", 1) + except (OSError, ValueError, UnicodeDecodeError): + return False + if prefix.casefold() != "gitdir": + return False + gitdir = Path(value.strip()) + if not gitdir.is_absolute(): + gitdir = marker.parent / gitdir + return gitdir.is_dir() + + def _prompts_dir_for(prompt_path: Path) -> Path: """Return the concrete prompts directory to pass into path resolution.""" return prompt_path.parent diff --git a/pdd/sync_core/reporting.py b/pdd/sync_core/reporting.py index db5988a8f..da8f75863 100644 --- a/pdd/sync_core/reporting.py +++ b/pdd/sync_core/reporting.py @@ -6,7 +6,7 @@ import subprocess from dataclasses import dataclass from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Iterable from .classifier import classify @@ -314,6 +314,7 @@ def build_canonical_report( unit for unit in manifest.managed_units if not wanted + or _module_identity(unit.unit_id.prompt_relpath) in wanted or unit.unit_id.prompt_relpath.stem.rsplit("_", 1)[0] in wanted or unit.unit_id.prompt_relpath.as_posix() in wanted ) @@ -344,3 +345,9 @@ def build_canonical_report( for verdict in verdicts ], } + + +def _module_identity(prompt_path: PurePosixPath) -> str: + """Return the prompt-root-relative, language-stripped module identity.""" + relative = prompt_path.relative_to("prompts") + return relative.with_suffix("").as_posix().rsplit("_", 1)[0] diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 92b9c84af..61bba76b5 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -9,6 +9,7 @@ import configparser import json import os +import platform import shlex import subprocess import sys @@ -328,6 +329,7 @@ def _transitive_support_blobs( included: set[PurePosixPath], test_artifacts: tuple[PurePosixPath, ...] = (), code_under_test_paths: frozenset[PurePosixPath] = frozenset(), + fail_on_dynamic: bool = False, ) -> tuple[tuple[PurePosixPath, bytes], ...]: # pylint: disable=too-many-arguments """Resolve local Python imports from protected runner support paths.""" @@ -343,13 +345,15 @@ def _transitive_support_blobs( source = read_git_blob(root, ref, path) if source is None or path.suffix != ".py": continue - discovered, _dynamic = _local_module_paths( + discovered, dynamic = _local_module_paths( root, ref, path, source, code_under_test_paths=code_under_test_paths, ) + if dynamic and fail_on_dynamic: + raise ValueError(f"unresolved dynamic product dependency: {path}") discovered -= visited paths.update(discovered) remaining.extend(discovered) @@ -459,7 +463,7 @@ def _support_digest( ) closure = _pytest_support_closure(root, ref, tests, product) product_closure = _transitive_support_blobs( - root, ref, pending=product, included=set(product) + root, ref, pending=product, included=set(product), fail_on_dynamic=True ) closure = tuple(sorted(set(closure + product_closure))) digest = hashlib.sha256() @@ -524,6 +528,8 @@ def runner_identity_digest( """Bind evidence to protected adapters, configs, and exact artifact scopes.""" payload = { "tool_version": TRUSTED_RUNNER_VERSION, + "python_runtime": _measured_python_runtime(), + "checker_artifact_digest": _checker_artifact_digest(), "pytest_command": [ "", "-m", @@ -567,6 +573,25 @@ def runner_identity_digest( ).hexdigest() +def _measured_python_runtime() -> dict[str, str]: + """Return stable runtime properties without installation-specific paths.""" + return { + "implementation": platform.python_implementation(), + "version": platform.python_version(), + "abi": getattr(sys.implementation, "cache_tag", ""), + "platform": sys.platform, + "machine": platform.machine(), + } + + +def _checker_artifact_digest() -> str: + """Hash the protected runner implementation that derives outcomes.""" + digest = hashlib.sha256() + for path in sorted((_CHECKER_PYTEST_PROBE, Path(__file__))): + digest.update(path.name.encode() + b"\0" + path.read_bytes() + b"\0") + return digest.hexdigest() + + def _changed_paths( root: Path, base_sha: str, @@ -683,7 +708,9 @@ def _trusted_collection_runner( ) -> tuple[Path, Path]: """Create a process-separated checker and candidate collection worker.""" worker = directory / "collection_worker.py" - worker_output = directory / f"collection-{os.urandom(16).hex()}.json" + result_dir = directory / f"results-{os.urandom(16).hex()}" + result_dir.mkdir(mode=0o700) + worker_output = result_dir / f"collection-{os.urandom(16).hex()}.json" worker_output.touch(mode=0o600) worker.write_text( "\n".join( @@ -732,7 +759,9 @@ def _trusted_execution_runner( ) -> tuple[Path, Path]: """Create a checker that never imports candidate code or pytest.""" worker = directory / "execution_worker.py" - worker_junit = directory / f"execution-{os.urandom(16).hex()}.xml" + result_dir = directory / f"results-{os.urandom(16).hex()}" + result_dir.mkdir(mode=0o700) + worker_junit = result_dir / f"execution-{os.urandom(16).hex()}.xml" worker_junit.touch(mode=0o600) worker.write_text( "\n".join(( diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 1469052f9..df7cb2912 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -5,18 +5,74 @@ import os import signal import subprocess +import sys +import time +import uuid +from pathlib import Path + + +def _marked_processes(token: str) -> set[int]: + """Return signer descendants carrying the per-call inherited marker.""" + marker = f"PDD_SIGNER_PROCESS_TOKEN={token}".encode() + found: set[int] = set() + if sys.platform.startswith("linux"): + entries = Path("/proc").iterdir() + for entry in entries: + if not entry.name.isdigit(): + continue + try: + values = (entry / "environ").read_bytes().split(b"\0") + except (OSError, PermissionError): + continue + if marker in values: + found.add(int(entry.name)) + return found + result = subprocess.run( + ["ps", "eww", "-axo", "pid=,command="], + capture_output=True, text=True, check=False, + ) + text_marker = marker.decode() + for line in result.stdout.splitlines(): + if text_marker not in line: + continue + try: + found.add(int(line.strip().split(None, 1)[0])) + except (ValueError, IndexError): + continue + return found + + +def _terminate_marked(token: str, leader: int, timeout: float = 0.5) -> None: + """Kill marked descendants across process groups within a fixed bound.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + pids = _marked_processes(token) - {os.getpid()} + if not pids: + return + for pid in pids: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + time.sleep(0.01) + try: + os.kill(leader, signal.SIGKILL) + except ProcessLookupError: + pass def run_signer( command: tuple[str, ...], payload: bytes, *, timeout: float ) -> subprocess.CompletedProcess[bytes]: """Run a signer in a new process group and reap the complete group on timeout.""" + token = uuid.uuid4().hex process = subprocess.Popen( # pylint: disable=consider-using-with command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True, + env=os.environ | {"PDD_SIGNER_PROCESS_TOKEN": token}, ) try: stdout, stderr = process.communicate(payload, timeout=timeout) @@ -25,7 +81,16 @@ def run_signer( os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass - stdout, stderr = process.communicate() + _terminate_marked(token, process.pid) + try: + stdout, stderr = process.communicate(timeout=0.5) + except subprocess.TimeoutExpired: + if process.stdout is not None: + process.stdout.close() + if process.stderr is not None: + process.stderr.close() + process.wait(timeout=0.5) + stdout, stderr = b"", b"" raise subprocess.TimeoutExpired( command, timeout, output=stdout, stderr=stderr ) from exc diff --git a/pdd/sync_core/transaction.py b/pdd/sync_core/transaction.py index 8092a401e..1b5ae2352 100644 --- a/pdd/sync_core/transaction.py +++ b/pdd/sync_core/transaction.py @@ -633,11 +633,25 @@ def recover(self, transaction_id: str) -> TransactionResult: item["installed"] = True changed.append(PurePosixPath(str(item["relpath"]))) self._write_journal(transaction_dir, payload) - except TransactionError: - self._restore_entries(transaction_dir, entries) + for item in entries: + if not isinstance(item, dict): + raise TransactionError("transaction entry is malformed") + relpath = PurePosixPath(str(item["relpath"])) + desired = FileState( + True, str(item["desired_digest"]), + str(item["desired_mode"]), "regular", + ) + if self._destination_state(relpath) != desired: + raise TransactionConflict(f"destination changed: {relpath}") + except TransactionError as exc: + try: + self._restore_entries(transaction_dir, entries) + except TransactionConflict: + self._write_journal(transaction_dir, payload) + raise payload["phase"] = TransactionPhase.ROLLED_BACK.value self._write_journal(transaction_dir, payload) - raise + raise exc payload["phase"] = TransactionPhase.COMMITTED.value self._write_journal(transaction_dir, payload) return TransactionResult( diff --git a/pdd/sync_orchestration.py b/pdd/sync_orchestration.py index 5db06ecd9..8e683aad2 100644 --- a/pdd/sync_orchestration.py +++ b/pdd/sync_orchestration.py @@ -2124,6 +2124,17 @@ def sync_orchestration( "operations_completed": [], "errors": [f"Path construction failed: {str(e)}"] } + + try: + from .sync_core.finalize import preflight_legacy_mutation + preflight_legacy_mutation(pdd_files) + except RuntimeError as exc: + return { + "success": False, + "error": str(exc), + "operations_completed": [], + "errors": [str(exc)], + } # Shared state for animation (passed to App) current_function_name_ref = ["initializing"] diff --git a/tests/test_sync_core_transaction.py b/tests/test_sync_core_transaction.py index 8c86ba50f..a17393589 100644 --- a/tests/test_sync_core_transaction.py +++ b/tests/test_sync_core_transaction.py @@ -152,7 +152,7 @@ def raced_install(transaction_dir, entry): target.write_text("external = True\n") monkeypatch.setattr(manager, "_install_entry", raced_install) - with pytest.raises(TransactionConflict, match="destination changed"): + with pytest.raises(TransactionConflict, match="rollback conflict"): manager.recover("tx-recovery-race") assert target.read_text() == "external = True\n" assert manager.incomplete() == ("tx-recovery-race",) From 21fffbe2386dd49e36bdcf783fab1f57bee72b77 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 00:39:35 -0700 Subject: [PATCH 084/128] test(sync): reproduce CLI preflight state writes --- tests/test_commands_maintenance.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_commands_maintenance.py b/tests/test_commands_maintenance.py index 378fd5290..7135dee5e 100644 --- a/tests/test_commands_maintenance.py +++ b/tests/test_commands_maintenance.py @@ -59,6 +59,22 @@ def test_sync_returns_tuple_on_success(mock_sync_main, mock_auto_update, runner) mock_sync_main.assert_called_once() +@patch("pdd.core.cli.auto_update") +@patch("pdd.commands.maintenance.sync_main") +@patch( + "pdd.sync_core.finalize.preflight_legacy_mutation", + side_effect=RuntimeError("protected canonical sync"), +) +def test_sync_cli_preflights_before_dispatch_and_suppresses_state_writes( + mock_preflight, mock_sync_main, mock_auto_update, runner +): + result = runner.invoke(cli.cli, ["--quiet", "sync", "test_module"]) + assert result.exit_code != 0 + mock_preflight.assert_called_once() + mock_sync_main.assert_not_called() + assert result.exception is not None + + @patch('pdd.core.cli.auto_update') @patch('pdd.commands.maintenance.sync_main') def test_sync_compress_flag_forwarded_to_sync_main( From d7a2f6651a9bdc0bb720c7fab83527d425f76357 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 00:40:54 -0700 Subject: [PATCH 085/128] fix(sync): preflight CLI before state bookkeeping --- pdd/commands/maintenance.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pdd/commands/maintenance.py b/pdd/commands/maintenance.py index b8ca34378..556204fa1 100644 --- a/pdd/commands/maintenance.py +++ b/pdd/commands/maintenance.py @@ -363,6 +363,15 @@ def _restore_model_default(_prev=_prev_model_default): effective_compressed_context = _resolve_compressed_context(compressed_context) ctx.obj["compressed_context"] = effective_compressed_context + if not dry_run: + from ..sync_core.finalize import preflight_legacy_mutation + + try: + preflight_legacy_mutation() + except RuntimeError as exc: + ctx.obj["_suppress_core_dump"] = True + raise click.UsageError(str(exc)) from exc + # No basename -> global Tier 1 sync if basename is None: if snapshot_context: From 51c2159fe212fffbc04670f0efcc9d7fdefb1251 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 01:22:37 -0700 Subject: [PATCH 086/128] test(sync): reproduce remaining high review boundaries --- tests/test_sync_core_runner.py | 92 ++++++++++++++++++++++++++++++++++ tests/test_sync_core_trust.py | 60 ++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index fe4fab59c..e64365df5 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -1,6 +1,7 @@ """Tests for pass-only trusted runner normalization and self-certification guards.""" import os +import shutil import subprocess import sys from datetime import datetime, timezone @@ -888,6 +889,33 @@ def test_candidate_cannot_forge_worker_outputs_discovered_from_script_path( assert executions[0].outcome is not EvidenceOutcome.PASS +@pytest.mark.skipif( + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux protected runner", +) +def test_candidate_cannot_recursively_forge_any_result_channel(tmp_path: Path) -> None: + content = ( + "import os, sys\nfrom pathlib import Path\n" + "trusted = Path(sys.argv[0]).resolve().parent\n" + "for path in trusted.rglob('*'):\n" + " if path.is_file() and (path.suffix in {'.json', '.xml'} or " + "path.name in {'node-ids.json', 'junit.xml'}):\n" + " try:\n" + " payload = ('[\"tests/test_widget.py::test_widget\"]' if " + "path.suffix == '.json' else " + "'')\n" + " path.write_text(payload)\n" + " except OSError:\n" + " pass\n" + "if Path(sys.argv[0]).name in {'collection_worker.py', 'execution_worker.py'}:\n" + " os._exit(0)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS + + def test_runner_identity_fails_closed_for_dynamic_product_loader(tmp_path: Path) -> None: root, _commit = _repository( tmp_path, "import product\ndef test_widget(): assert product.value == 1\n" @@ -904,6 +932,36 @@ def test_runner_identity_fails_closed_for_dynamic_product_loader(tmp_path: Path) runner_identity_digest(profile, root=root, ref=commit) +@pytest.mark.parametrize( + "product", + [ + "loader = __builtins__['__import__']\nvalue = loader('helper').value\n", + "loader = getattr(__builtins__, '__getitem__')('__import__')\n" + "value = loader('helper').value\n", + ], +) +def test_runner_identity_fails_closed_for_aliased_reflective_loader( + tmp_path: Path, product: str +) -> None: + root, _commit = _repository( + tmp_path, "import product\ndef test_widget(): assert product.value == 1\n" + ) + (root / "product.py").write_text(product) + (root / "helper.py").write_text("value = 1\n") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "aliased product loader") + before = _git(root, "rev-parse", "HEAD") + profile = _profile(root, before, (PurePosixPath("product.py"),)) + before_digest = runner_identity_digest(profile, root=root, ref=before) + (root / "helper.py").write_text("value = 2\n") + _git(root, "add", "helper.py") + _git(root, "commit", "-q", "-m", "change hidden helper") + after = _git(root, "rev-parse", "HEAD") + with pytest.raises(ValueError, match="dynamic product dependency"): + runner_identity_digest(profile, root=root, ref=after) + assert before_digest + + def test_runner_identity_binds_measured_runtime_and_checker( tmp_path: Path, monkeypatch ) -> None: @@ -946,6 +1004,40 @@ def test_runner_identity_is_stable_across_interpreter_prefixes( assert runner_identity_digest(profile, root=root, ref=commit) == first +def test_runner_identity_binds_actual_supervisor_bytes(tmp_path: Path, monkeypatch) -> None: + root, commit = _repository(tmp_path, "def test_widget(): assert True\n") + profile = _profile(root, commit) + first = runner_identity_digest(profile, root=root, ref=commit) + source = Path(__file__).parents[1] / "pdd/sync_core/supervisor.py" + changed = tmp_path / "supervisor.py" + changed.write_bytes(source.read_bytes() + b"\n# changed checker semantics\n") + monkeypatch.setattr("pdd.sync_core.supervisor.__file__", str(changed)) + assert runner_identity_digest(profile, root=root, ref=commit) != first + + +def test_runner_identity_binds_actual_pytest_bytes_and_ignores_prefix( + tmp_path: Path, monkeypatch +) -> None: + root, commit = _repository(tmp_path, "def test_widget(): assert True\n") + profile = _profile(root, commit) + pytest_source = Path(pytest.__file__).resolve() + first_prefix = tmp_path / "venv-a" + second_prefix = tmp_path / "venv-b" + relative = Path("lib/python/site-packages/pytest/__init__.py") + first_file = first_prefix / relative + second_file = second_prefix / relative + first_file.parent.mkdir(parents=True) + second_file.parent.mkdir(parents=True) + first_file.write_bytes(pytest_source.read_bytes()) + second_file.write_bytes(pytest_source.read_bytes()) + monkeypatch.setattr(pytest, "__file__", str(first_file)) + first = runner_identity_digest(profile, root=root, ref=commit) + monkeypatch.setattr(pytest, "__file__", str(second_file)) + assert runner_identity_digest(profile, root=root, ref=commit) == first + second_file.write_bytes(second_file.read_bytes() + b"\n# changed pytest semantics\n") + assert runner_identity_digest(profile, root=root, ref=commit) != first + + def test_pytest_plugin_guard_ignores_non_pytest_preview_prose(tmp_path: Path) -> None: root, _commit = _repository(tmp_path, "def test_widget(): assert True\n") (root / "setup.cfg").write_text( diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 73306fbc0..d5dac0582 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -2,6 +2,8 @@ import base64 import json +import os +from pathlib import Path import subprocess import sys import time @@ -25,6 +27,9 @@ from pdd.sync_core.trust import ValidationEvidence from pdd.sync_core.finalize import attestation_signer_from_environment from pdd.sync_core.signer_process import run_signer +from pdd.sync_core.certificate import RemoteCertificateSigner +import pdd.sync_core.certificate as certificate_module +import pdd.sync_core.trust as trust_module from pdd.sync_core.types import ObligationEvidence @@ -134,6 +139,61 @@ def test_signer_timeout_is_bounded_with_detached_pipe_holder() -> None: assert time.monotonic() - started < 1.5 +@pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="requires Linux stable process identity" +) +@pytest.mark.parametrize("adapter", ["attestation", "certificate"]) +def test_remote_signer_timeout_reaps_env_cleared_detached_descendant( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, adapter: str +) -> None: + pid_file = tmp_path / f"{adapter}.pid" + marker = tmp_path / f"{adapter}.survived" + detached = ( + "import os,sys,time; " + "open(sys.argv[1], 'w').write(str(os.getpid())); " + "time.sleep(1); open(sys.argv[2], 'w').write('survived'); time.sleep(30)" + ) + parent = ( + "import os,subprocess,sys,time; env=dict(os.environ); " + "env.pop('PDD_SIGNER_PROCESS_TOKEN', None); " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2], sys.argv[3]], " + "env=env, start_new_session=True, stdout=subprocess.DEVNULL, " + "stderr=subprocess.DEVNULL); time.sleep(30)" + ) + command = (sys.executable, "-c", parent, detached, str(pid_file), str(marker)) + actual_run_signer = run_signer + + def short_run(command_value, payload, *, timeout): + del timeout + return actual_run_signer(command_value, payload, timeout=0.1) + + if adapter == "attestation": + monkeypatch.setattr(trust_module, "run_signer", short_run) + signer = RemoteAttestationSigner(SIGNER.issuer, PUBLIC_KEY, command) + with pytest.raises(AttestationError, match="timed out"): + signer.issue( + AttestationRequest( + "remote-attestation", _binding(), + (ObligationEvidence("test", EvidenceOutcome.PASS),), + "remote-nonce", NOW, + ) + ) + else: + monkeypatch.setattr(certificate_module, "run_signer", short_run) + signer = RemoteCertificateSigner(SIGNER.issuer, PUBLIC_KEY, command) + with pytest.raises(ValueError, match="timed out"): + signer.sign_bytes(b"payload") + + deadline = time.monotonic() + 1 + while not pid_file.exists() and time.monotonic() < deadline: + time.sleep(0.01) + pid = int(pid_file.read_text(encoding="utf-8")) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + time.sleep(1.1) + assert not marker.exists() + + def test_attestation_environment_forbids_local_private_key(monkeypatch) -> None: monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-secret") with pytest.raises(ValueError, match="local attestation signing keys are forbidden"): From e281fa7f0f13c26e3a56394def120792794b194e Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 01:28:10 -0700 Subject: [PATCH 087/128] fix(sync): close remaining high review boundaries --- pdd/sync_core/runner.py | 81 ++++++++++++++++++++++++++------- pdd/sync_core/signer_process.py | 57 ++++++++++++++++++++--- tests/test_sync_core_runner.py | 4 +- 3 files changed, 117 insertions(+), 25 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 61bba76b5..2d9e64dcd 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -16,10 +16,13 @@ import tempfile import tomllib import xml.etree.ElementTree as ET +import importlib.metadata from dataclasses import dataclass from datetime import datetime from pathlib import Path, PurePosixPath +import pytest + from .trust import ( AttestationBinding, AttestationEnvelope, @@ -465,6 +468,16 @@ def _support_digest( product_closure = _transitive_support_blobs( root, ref, pending=product, included=set(product), fail_on_dynamic=True ) + if product: + # Product code is an arbitrary Python runtime. Measuring every committed + # Python module is the only static closure that remains complete under + # loader aliasing and reflection without executing candidate code. + repository_python = tuple( + (path, blob) + for path in _git_python_paths(root, ref) + if (blob := read_git_blob(root, ref, path)) is not None + ) + product_closure = tuple(sorted(set(product_closure + repository_python))) closure = tuple(sorted(set(closure + product_closure))) digest = hashlib.sha256() for path, content in closure: @@ -585,13 +598,42 @@ def _measured_python_runtime() -> dict[str, str]: def _checker_artifact_digest() -> str: - """Hash the protected runner implementation that derives outcomes.""" + """Hash checker modules and locked runtime dependency bytes by logical name.""" digest = hashlib.sha256() - for path in sorted((_CHECKER_PYTEST_PROBE, Path(__file__))): - digest.update(path.name.encode() + b"\0" + path.read_bytes() + b"\0") + from . import isolation, supervisor # pylint: disable=import-outside-toplevel + + modules = { + "pdd/sync_core/runner.py": Path(__file__), + "pdd/sync_core/pytest_probe.py": _CHECKER_PYTEST_PROBE, + "pdd/sync_core/supervisor.py": Path(supervisor.__file__), + "pdd/sync_core/isolation.py": Path(isolation.__file__), + "pytest/__init__.py": Path(pytest.__file__), + } + for name, path in sorted(modules.items()): + digest.update(name.encode() + b"\0" + path.read_bytes() + b"\0") + for distribution_name in ("pytest", "pluggy", "iniconfig", "packaging"): + distribution = importlib.metadata.distribution(distribution_name) + digest.update(distribution_name.encode() + b"\0") + files = distribution.files or () + for member in sorted(files, key=str): + path = Path(distribution.locate_file(member)) + if path.is_file() and not path.is_symlink(): + digest.update(str(member).encode() + b"\0" + path.read_bytes() + b"\0") return digest.hexdigest() +def _git_python_paths(root: Path, ref: str) -> tuple[PurePosixPath, ...]: + """Return every committed Python path at an exact ref.""" + result = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", ref], cwd=root, + capture_output=True, text=True, check=True, + ) + return tuple( + PurePosixPath(value) for value in result.stdout.splitlines() + if value.endswith(".py") + ) + + def _changed_paths( root: Path, base_sha: str, @@ -708,7 +750,7 @@ def _trusted_collection_runner( ) -> tuple[Path, Path]: """Create a process-separated checker and candidate collection worker.""" worker = directory / "collection_worker.py" - result_dir = directory / f"results-{os.urandom(16).hex()}" + result_dir = directory.parent / f"results-{os.urandom(16).hex()}" result_dir.mkdir(mode=0o700) worker_output = result_dir / f"collection-{os.urandom(16).hex()}.json" worker_output.touch(mode=0o600) @@ -735,7 +777,8 @@ def _trusted_collection_runner( "import pytest", "if _ROOT not in sys.path:", " sys.path.insert(0, _ROOT)", - "raise SystemExit(pytest.main(" + json.dumps(pytest_args) + ", plugins=[_MODULE]))", + "_STATUS = pytest.main(" + json.dumps(pytest_args) + ", plugins=[_MODULE])", + "os._exit(80 + int(_STATUS))", "", ) ), @@ -747,7 +790,8 @@ def _trusted_collection_runner( "import pathlib, subprocess, sys", f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", f"source = pathlib.Path({json.dumps(str(worker_output))})", - "if result.returncode not in (0, 5) or not source.stat().st_size: raise SystemExit(1)", + "if result.returncode not in (80, 85): raise SystemExit(1)", + "if not source.stat().st_size: raise SystemExit(1)", f"pathlib.Path({json.dumps(str(collection_output))}).write_bytes(source.read_bytes())", "",)), encoding="utf-8", ) @@ -759,7 +803,7 @@ def _trusted_execution_runner( ) -> tuple[Path, Path]: """Create a checker that never imports candidate code or pytest.""" worker = directory / "execution_worker.py" - result_dir = directory / f"results-{os.urandom(16).hex()}" + result_dir = directory.parent / f"results-{os.urandom(16).hex()}" result_dir.mkdir(mode=0o700) worker_junit = result_dir / f"execution-{os.urandom(16).hex()}.xml" worker_junit.touch(mode=0o600) @@ -768,8 +812,9 @@ def _trusted_execution_runner( "import os, sys", "import pytest", f"os.chdir({json.dumps(str(root))})", f"sys.path.insert(0, {json.dumps(str(root))})", - "raise SystemExit(pytest.main(" + - json.dumps(pytest_args + [f"--junitxml={worker_junit}"]) + "))", "", + "_STATUS = pytest.main(" + + json.dumps(pytest_args + [f"--junitxml={worker_junit}"]) + ")", + "os._exit(80 + int(_STATUS))", "", )), encoding="utf-8", ) checker = directory / "run_execution.py" @@ -780,7 +825,7 @@ def _trusted_execution_runner( f"source = pathlib.Path({json.dumps(str(worker_junit))})", "if not source.stat().st_size: raise SystemExit(1)", f"pathlib.Path({json.dumps(str(junit))}).write_bytes(source.read_bytes())", - "raise SystemExit(result.returncode)", + "raise SystemExit(result.returncode - 80 if 80 <= result.returncode <= 85 else 1)", "",)), encoding="utf-8", ) return checker, worker_junit @@ -804,15 +849,17 @@ def _run_test_node( ).hexdigest() with tempfile.TemporaryDirectory(prefix="pdd-trusted-runner-") as directory: temporary = Path(directory) + controllers = temporary / f"controller-{os.urandom(16).hex()}" + controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) - junit = temporary / "junit.xml" + junit = temporary / f"authoritative-{os.urandom(16).hex()}.xml" junit.touch(mode=0o600) controller, worker_junit = _trusted_execution_runner( - temporary, root, command[3:], junit + controllers, root, command[3:], junit ) result, surviving = _managed_subprocess( - [sys.executable, str(controller)], cwd=temporary, + [sys.executable, str(controller)], cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), writable_files=(junit, worker_junit), readable_roots=(root,), @@ -847,9 +894,11 @@ def _collect_node_ids( """Collect exact pytest node IDs through the protected adapter.""" with tempfile.TemporaryDirectory(prefix="pdd-trusted-collection-") as directory: temporary = Path(directory) + controllers = temporary / f"controller-{os.urandom(16).hex()}" + controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) - collection_output = temporary / "node-ids.json" + collection_output = temporary / f"authoritative-{os.urandom(16).hex()}.json" collection_output.touch(mode=0o600) pytest_args = [ "--collect-only", @@ -861,7 +910,7 @@ def _collect_node_ids( path.as_posix(), ] runner, worker_output = _trusted_collection_runner( - temporary, root, pytest_args, collection_output + controllers, root, pytest_args, collection_output ) command = [sys.executable, str(runner)] digest = hashlib.sha256( @@ -877,7 +926,7 @@ def _collect_node_ids( ).encode() ).hexdigest() result, surviving = _managed_subprocess( - command, cwd=temporary, timeout=timeout_seconds, + command, cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), writable_files=(collection_output, worker_output), readable_roots=(root, _CHECKER_PYTEST_PROBE), diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index df7cb2912..ac522492a 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -11,6 +11,45 @@ from pathlib import Path +_LINUX_CONTAINMENT = r""" +import ctypes, os, signal, subprocess, sys, time +ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) +child = None +def descendants(root): + children = {} + for name in os.listdir('/proc'): + if not name.isdigit(): continue + try: + fields = open('/proc/' + name + '/stat').read().rsplit(')', 1)[1].split() + children.setdefault(int(fields[1]), set()).add(int(name)) + except (OSError, ValueError, IndexError): pass + found, pending = set(), [root] + while pending: + for pid in children.get(pending.pop(), ()): + if pid not in found: found.add(pid); pending.append(pid) + return found +def stop(_signum, _frame): + if child is not None: + deadline = time.monotonic() + .4 + while time.monotonic() < deadline: + pids = descendants(os.getpid()) | {child.pid} + if not pids: break + for pid in pids: + try: os.kill(pid, signal.SIGKILL) + except ProcessLookupError: pass + time.sleep(.01) + try: child.wait(timeout=.1) + except (subprocess.TimeoutExpired, ChildProcessError): pass + raise SystemExit(124) +signal.signal(signal.SIGTERM, stop) +child = subprocess.Popen(sys.argv[1:], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) +stdout, stderr = child.communicate(sys.stdin.buffer.read()) +sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) +raise SystemExit(child.returncode) +""" + + def _marked_processes(token: str) -> set[int]: """Return signer descendants carrying the per-call inherited marker.""" marker = f"PDD_SIGNER_PROCESS_TOKEN={token}".encode() @@ -66,8 +105,11 @@ def run_signer( ) -> subprocess.CompletedProcess[bytes]: """Run a signer in a new process group and reap the complete group on timeout.""" token = uuid.uuid4().hex + contained_command = command + if sys.platform.startswith("linux"): + contained_command = (sys.executable, "-c", _LINUX_CONTAINMENT, *command) process = subprocess.Popen( # pylint: disable=consider-using-with - command, + contained_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -77,11 +119,14 @@ def run_signer( try: stdout, stderr = process.communicate(payload, timeout=timeout) except subprocess.TimeoutExpired as exc: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - _terminate_marked(token, process.pid) + if sys.platform.startswith("linux"): + process.terminate() + else: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + _terminate_marked(token, process.pid) try: stdout, stderr = process.communicate(timeout=0.5) except subprocess.TimeoutExpired: diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index e64365df5..ccf4768d9 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -957,9 +957,7 @@ def test_runner_identity_fails_closed_for_aliased_reflective_loader( _git(root, "add", "helper.py") _git(root, "commit", "-q", "-m", "change hidden helper") after = _git(root, "rev-parse", "HEAD") - with pytest.raises(ValueError, match="dynamic product dependency"): - runner_identity_digest(profile, root=root, ref=after) - assert before_digest + assert runner_identity_digest(profile, root=root, ref=after) != before_digest def test_runner_identity_binds_measured_runtime_and_checker( From d03d4cf762304c559a145b9a59e44e581ebf8bb1 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 01:32:14 -0700 Subject: [PATCH 088/128] fix(sync): preserve protected pytest diagnostics --- pdd/sync_core/runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 2d9e64dcd..8f5d00024 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -778,6 +778,7 @@ def _trusted_collection_runner( "if _ROOT not in sys.path:", " sys.path.insert(0, _ROOT)", "_STATUS = pytest.main(" + json.dumps(pytest_args) + ", plugins=[_MODULE])", + "sys.stdout.flush(); sys.stderr.flush()", "os._exit(80 + int(_STATUS))", "", ) @@ -814,6 +815,7 @@ def _trusted_execution_runner( f"sys.path.insert(0, {json.dumps(str(root))})", "_STATUS = pytest.main(" + json.dumps(pytest_args + [f"--junitxml={worker_junit}"]) + ")", + "sys.stdout.flush(); sys.stderr.flush()", "os._exit(80 + int(_STATUS))", "", )), encoding="utf-8", ) From dd87f77d5db72c4ae3a984271a5f41abff1ee439 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:08:20 -0700 Subject: [PATCH 089/128] test(sync): cover complete trusted runtime boundaries --- tests/test_sync_core_runner.py | 95 ++++++++++++++++++++++++++++++++++ tests/test_sync_core_trust.py | 80 ++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index ccf4768d9..fa32e9875 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -8,6 +8,7 @@ from pathlib import Path, PurePosixPath import pytest +import pdd.sync_core.runner as runner_module from pdd.sync_core import ( AttestationSigner, @@ -889,6 +890,31 @@ def test_candidate_cannot_forge_worker_outputs_discovered_from_script_path( assert executions[0].outcome is not EvidenceOutcome.PASS +@pytest.mark.skipif( + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux protected runner", +) +def test_candidate_reading_worker_source_cannot_forge_collection_or_execution( + tmp_path: Path, +) -> None: + """A worker script is candidate-readable, so it cannot name an authority.""" + content = ( + "import os, re, sys\nfrom pathlib import Path\n" + "source = Path(sys.argv[0]).read_text(encoding='utf-8')\n" + "for name in re.findall(r'/[^\\\" ]*/results-[0-9a-f]+/" + "(?:collection|execution)-[0-9a-f]+\\.(?:json|xml)', source):\n" + " payload = ('[\\\"tests/test_widget.py::test_widget\\\"]' if name.endswith('.json') " + "else '')\n" + " Path(name).write_text(payload, encoding='utf-8')\n" + "if Path(sys.argv[0]).name in {'collection_worker.py', 'execution_worker.py'}:\n" + " os._exit(80)\n" + "def test_widget(): assert False\n" + ) + root, commit = _repository(tmp_path, content) + _envelope, executions = _run(root, commit, commit) + assert executions[0].outcome is not EvidenceOutcome.PASS + + @pytest.mark.skipif( not sys.platform.startswith("linux") or not shutil.which("bwrap"), reason="requires Linux protected runner", @@ -991,6 +1017,75 @@ def test_runner_identity_binds_transitive_product_dependency(tmp_path: Path) -> assert runner_identity_digest(profile, root=root, ref=after) != before_digest +def test_runner_identity_binds_product_readable_repository_data(tmp_path: Path) -> None: + root, _commit = _repository( + tmp_path, + "import product\ndef test_widget(): assert product.value == 1\n", + ) + (root / "product.py").write_text( + "import json\nvalue = json.loads(open('helper.json').read())['value']\n", + encoding="utf-8", + ) + (root / "helper.json").write_text('{"value": 1}\n', encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-q", "-m", "product data closure") + before = _git(root, "rev-parse", "HEAD") + profile = _profile(root, before, (PurePosixPath("product.py"),)) + first = runner_identity_digest(profile, root=root, ref=before) + (root / "helper.json").write_text('{"value": 2}\n', encoding="utf-8") + _git(root, "add", "helper.json") + _git(root, "commit", "-q", "-m", "change product data") + after = _git(root, "rev-parse", "HEAD") + assert runner_identity_digest(profile, root=root, ref=after) != first + + +def test_released_runtime_digest_binds_installed_native_dependency( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + native = tmp_path / "site-packages" / "native_extension.so" + native.parent.mkdir() + native.write_bytes(b"native-v1") + monkeypatch.setattr( + runner_module, + "_released_runtime_closure_paths", + lambda: (("python-runtime/site-packages/native_extension.so", native),), + raising=False, + ) + first = runner_module._released_runtime_closure_digest() + native.write_bytes(b"native-v2") + assert runner_module._released_runtime_closure_digest() != first + + +def test_released_runtime_digest_binds_runtime_and_sandbox_bytes_prefix_portably( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + first_prefix = tmp_path / "prefix-a" + second_prefix = tmp_path / "prefix-b" + first_python = first_prefix / "bin/python" + second_python = second_prefix / "bin/python" + first_bwrap = first_prefix / "bin/bwrap" + second_bwrap = second_prefix / "bin/bwrap" + for path in (first_python, second_python, first_bwrap, second_bwrap): + path.parent.mkdir(parents=True, exist_ok=True) + first_python.write_bytes(b"python-runtime") + second_python.write_bytes(b"python-runtime") + first_bwrap.write_bytes(b"sandbox-runtime") + second_bwrap.write_bytes(b"sandbox-runtime") + paths = (("interpreter/python", first_python), ("sandbox/bwrap", first_bwrap)) + monkeypatch.setattr( + runner_module, "_released_runtime_closure_paths", lambda: paths, raising=False + ) + first = runner_module._released_runtime_closure_digest() + monkeypatch.setattr( + runner_module, + "_released_runtime_closure_paths", + lambda: (("interpreter/python", second_python), ("sandbox/bwrap", second_bwrap)), + ) + assert runner_module._released_runtime_closure_digest() == first + second_bwrap.write_bytes(b"changed sandbox runtime") + assert runner_module._released_runtime_closure_digest() != first + + def test_runner_identity_is_stable_across_interpreter_prefixes( tmp_path: Path, monkeypatch ) -> None: diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index d5dac0582..6a21b920b 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -194,6 +194,86 @@ def short_run(command_value, payload, *, timeout): assert not marker.exists() +@pytest.mark.skipif( + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux PID namespace containment", +) +@pytest.mark.parametrize("adapter", ["attestation", "certificate"]) +def test_remote_signer_normal_return_reaps_detached_descendant( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, adapter: str +) -> None: + marker = tmp_path / f"{adapter}.normal-return-leak" + child = ( + "import os,sys,time; os.setsid(); time.sleep(.5); " + "open(sys.argv[1], 'w').write('leaked')" + ) + parent = ( + "import os,subprocess,sys; env=dict(os.environ); " + "env.pop('PDD_SIGNER_PROCESS_TOKEN', None); " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " + "env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); " + "sys.stdout.buffer.write(b'not-a-signature')" + ) + command = (sys.executable, "-c", parent, child, str(marker)) + if adapter == "attestation": + signer = RemoteAttestationSigner(SIGNER.issuer, PUBLIC_KEY, command) + with pytest.raises(AttestationError): + signer.issue(AttestationRequest("remote", _binding(), (), "nonce", NOW)) + else: + signer = RemoteCertificateSigner(SIGNER.issuer, PUBLIC_KEY, command) + with pytest.raises(ValueError): + signer.sign_bytes(b"payload") + time.sleep(.6) + assert not marker.exists() + + +@pytest.mark.skipif( + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux PID namespace containment", +) +@pytest.mark.parametrize("adapter", ["attestation", "certificate"]) +@pytest.mark.parametrize("leader_signal", ["SIGSTOP", "SIGKILL"]) +def test_remote_signer_leader_loss_reaps_detached_descendant( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + adapter: str, + leader_signal: str, +) -> None: + marker = tmp_path / f"{adapter}-{leader_signal}.leader-leak" + child = ( + "import os,sys,time; os.setsid(); time.sleep(.5); " + "open(sys.argv[1], 'w').write('leaked')" + ) + parent = ( + "import os,signal,subprocess,sys,time; env=dict(os.environ); " + "env.pop('PDD_SIGNER_PROCESS_TOKEN', None); " + "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " + "env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); " + "os.kill(os.getppid(), getattr(signal, sys.argv[3])); time.sleep(30)" + ) + command = (sys.executable, "-c", parent, child, str(marker), leader_signal) + actual_run_signer = run_signer + + def short_run(command_value, payload, *, timeout): + del timeout + return actual_run_signer(command_value, payload, timeout=0.1) + + started = time.monotonic() + if adapter == "attestation": + monkeypatch.setattr(trust_module, "run_signer", short_run) + signer = RemoteAttestationSigner(SIGNER.issuer, PUBLIC_KEY, command) + with pytest.raises(AttestationError): + signer.issue(AttestationRequest("remote", _binding(), (), "nonce", NOW)) + else: + monkeypatch.setattr(certificate_module, "run_signer", short_run) + signer = RemoteCertificateSigner(SIGNER.issuer, PUBLIC_KEY, command) + with pytest.raises(ValueError): + signer.sign_bytes(b"payload") + assert time.monotonic() - started < 1.5 + time.sleep(.6) + assert not marker.exists() + + def test_attestation_environment_forbids_local_private_key(monkeypatch) -> None: monkeypatch.setenv("PDD_ATTESTATION_SIGNING_KEY", "candidate-secret") with pytest.raises(ValueError, match="local attestation signing keys are forbidden"): From 1d5cf322bafec9414b924869dc4a7d19a546603f Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:15:04 -0700 Subject: [PATCH 090/128] fix(sync): bind protected runtime and signer containment --- pdd/sync_core/runner.py | 164 +++++++++++++------------------- pdd/sync_core/signer_process.py | 98 ++++++++++++------- pdd/sync_core/supervisor.py | 91 ++++++++++++++++-- tests/test_sync_core_trust.py | 3 +- 4 files changed, 214 insertions(+), 142 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 8f5d00024..0256f63a1 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -38,7 +38,7 @@ VerificationObligation, VerificationProfile, ) -from .supervisor import run_supervised +from .supervisor import released_runtime_closure_paths, run_supervised TRUSTED_RUNNER_VERSION = "pdd-trusted-runner-v2" @@ -469,15 +469,14 @@ def _support_digest( root, ref, pending=product, included=set(product), fail_on_dynamic=True ) if product: - # Product code is an arbitrary Python runtime. Measuring every committed - # Python module is the only static closure that remains complete under - # loader aliasing and reflection without executing candidate code. - repository_python = tuple( + # Product code can read every committed artifact in its exact-head + # checkout, not only importable Python source. + repository_artifacts = tuple( (path, blob) - for path in _git_python_paths(root, ref) + for path in _git_paths(root, ref) if (blob := read_git_blob(root, ref, path)) is not None ) - product_closure = tuple(sorted(set(product_closure + repository_python))) + product_closure = tuple(sorted(set(product_closure + repository_artifacts))) closure = tuple(sorted(set(closure + product_closure))) digest = hashlib.sha256() for path, content in closure: @@ -542,6 +541,7 @@ def runner_identity_digest( payload = { "tool_version": TRUSTED_RUNNER_VERSION, "python_runtime": _measured_python_runtime(), + "released_runtime_digest": _released_runtime_closure_digest(), "checker_artifact_digest": _checker_artifact_digest(), "pytest_command": [ "", @@ -622,15 +622,34 @@ def _checker_artifact_digest() -> str: return digest.hexdigest() -def _git_python_paths(root: Path, ref: str) -> tuple[PurePosixPath, ...]: - """Return every committed Python path at an exact ref.""" +def _released_runtime_closure_paths() -> tuple[tuple[str, Path], ...]: + """Return the complete, logically named runtime exposed to protected pytest.""" + paths = list(released_runtime_closure_paths()) + paths.extend(( + ("checker/pdd/sync_core/runner.py", Path(__file__)), + ("checker/pdd/sync_core/pytest_probe.py", _CHECKER_PYTEST_PROBE), + )) + return tuple(sorted(paths, key=lambda item: item[0])) + + +def _released_runtime_closure_digest() -> str: + """Hash the released runtime by logical name, never installation prefix.""" + digest = hashlib.sha256() + for name, path in _released_runtime_closure_paths(): + if not path.is_file() or path.is_symlink(): + raise RuntimeError(f"released runtime entry is not a regular file: {name}") + digest.update(name.encode("utf-8") + b"\0" + path.read_bytes() + b"\0") + return digest.hexdigest() + + +def _git_paths(root: Path, ref: str) -> tuple[PurePosixPath, ...]: + """Return every committed regular path at an exact ref.""" result = subprocess.run( ["git", "ls-tree", "-r", "--name-only", ref], cwd=root, capture_output=True, text=True, check=True, ) return tuple( PurePosixPath(value) for value in result.stdout.splitlines() - if value.endswith(".py") ) @@ -746,91 +765,59 @@ def _trusted_collection_runner( directory: Path, root: Path, pytest_args: list[str], - collection_output: Path, -) -> tuple[Path, Path]: - """Create a process-separated checker and candidate collection worker.""" + ) -> Path: + """Create a worker that cannot see an authoritative result channel.""" worker = directory / "collection_worker.py" - result_dir = directory.parent / f"results-{os.urandom(16).hex()}" - result_dir.mkdir(mode=0o700) - worker_output = result_dir / f"collection-{os.urandom(16).hex()}.json" - worker_output.touch(mode=0o600) worker.write_text( "\n".join( ( - "import importlib.util", - "import os", - "import sys", + "import os, subprocess, sys", "", f"_ROOT = {json.dumps(str(root))}", - f"_PROBE = {json.dumps(str(_CHECKER_PYTEST_PROBE))}", "", "os.chdir(_ROOT)", - "_SPEC = importlib.util.spec_from_file_location(", - " '_pdd_checker_pytest_probe_abs', _PROBE", - ")", - "if _SPEC is None or _SPEC.loader is None:", - " raise ImportError('checker pytest probe is unavailable')", - "_MODULE = importlib.util.module_from_spec(_SPEC)", - "_SPEC.loader.exec_module(_MODULE)", - f"_MODULE._OUTPUT_PATH = {json.dumps(str(worker_output))}", - "", - "import pytest", "if _ROOT not in sys.path:", " sys.path.insert(0, _ROOT)", - "_STATUS = pytest.main(" + json.dumps(pytest_args) + ", plugins=[_MODULE])", + "_STATUS = subprocess.run([sys.executable, '-m', 'pytest'] + " + + json.dumps(pytest_args) + ").returncode", "sys.stdout.flush(); sys.stderr.flush()", - "os._exit(80 + int(_STATUS))", + "os._exit(_STATUS if _STATUS in (0, 5) else 125)", "", ) ), encoding="utf-8", ) - checker = directory / "run_collection.py" - checker.write_text( - "\n".join(( - "import pathlib, subprocess, sys", - f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", - f"source = pathlib.Path({json.dumps(str(worker_output))})", - "if result.returncode not in (80, 85): raise SystemExit(1)", - "if not source.stat().st_size: raise SystemExit(1)", - f"pathlib.Path({json.dumps(str(collection_output))}).write_bytes(source.read_bytes())", - "",)), encoding="utf-8", - ) - return checker, worker_output + return worker def _trusted_execution_runner( - directory: Path, root: Path, pytest_args: list[str], junit: Path -) -> tuple[Path, Path]: - """Create a checker that never imports candidate code or pytest.""" + directory: Path, root: Path, pytest_args: list[str] +) -> Path: + """Create a worker whose raw pytest status is normalized outside the sandbox.""" worker = directory / "execution_worker.py" - result_dir = directory.parent / f"results-{os.urandom(16).hex()}" - result_dir.mkdir(mode=0o700) - worker_junit = result_dir / f"execution-{os.urandom(16).hex()}.xml" - worker_junit.touch(mode=0o600) worker.write_text( "\n".join(( - "import os, sys", "import pytest", + "import os, subprocess, sys", f"os.chdir({json.dumps(str(root))})", f"sys.path.insert(0, {json.dumps(str(root))})", - "_STATUS = pytest.main(" + - json.dumps(pytest_args + [f"--junitxml={worker_junit}"]) + ")", + "_STATUS = subprocess.run([sys.executable, '-m', 'pytest'] + " + + json.dumps(pytest_args) + ").returncode", "sys.stdout.flush(); sys.stderr.flush()", - "os._exit(80 + int(_STATUS))", "", + "os._exit(_STATUS if 0 <= _STATUS <= 5 else 125)", "", )), encoding="utf-8", ) - checker = directory / "run_execution.py" - checker.write_text( - "\n".join(( - "import pathlib, subprocess, sys", - f"result = subprocess.run([sys.executable, {json.dumps(str(worker))}])", - f"source = pathlib.Path({json.dumps(str(worker_junit))})", - "if not source.stat().st_size: raise SystemExit(1)", - f"pathlib.Path({json.dumps(str(junit))}).write_bytes(source.read_bytes())", - "raise SystemExit(result.returncode - 80 if 80 <= result.returncode <= 85 else 1)", - "",)), encoding="utf-8", - ) - return checker, worker_junit + return worker + + +def _pytest_exit_outcome(returncode: int, output: str) -> tuple[EvidenceOutcome, str]: + """Normalize the worker's whitelisted pytest status outside candidate control.""" + if returncode == 0: + return EvidenceOutcome.PASS, "protected pytest completed without failures" + if returncode == 1: + return EvidenceOutcome.FAIL, "protected pytest reported test failures" + if returncode == 5: + return EvidenceOutcome.NOT_COLLECTED, "protected pytest collected no tests" + return EvidenceOutcome.ERROR, "protected pytest failed: " + output[-500:] def _run_test_node( @@ -855,15 +842,11 @@ def _run_test_node( controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) - junit = temporary / f"authoritative-{os.urandom(16).hex()}.xml" - junit.touch(mode=0o600) - controller, worker_junit = _trusted_execution_runner( - controllers, root, command[3:], junit - ) + worker = _trusted_execution_runner(controllers, root, command[3:]) result, surviving = _managed_subprocess( - [sys.executable, str(controller)], cwd=controllers, + [sys.executable, str(worker)], cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), - writable_roots=(home.parent,), writable_files=(junit, worker_junit), + writable_roots=(home.parent,), readable_roots=(root,), ) if result.returncode == 124: @@ -878,11 +861,8 @@ def _run_test_node( node_id, EvidenceOutcome.ERROR, command_digest, "validator left a surviving process-group descendant", ) - outcome, detail = _junit_outcome( - junit, - result.returncode, - result.stdout + "\n" + result.stderr, - 1, + outcome, detail = _pytest_exit_outcome( + result.returncode, result.stdout + "\n" + result.stderr ) return RunnerExecution(node_id, outcome, command_digest, detail) @@ -900,8 +880,6 @@ def _collect_node_ids( controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) - collection_output = temporary / f"authoritative-{os.urandom(16).hex()}.json" - collection_output.touch(mode=0o600) pytest_args = [ "--collect-only", "-q", @@ -911,10 +889,8 @@ def _collect_node_ids( "no:cacheprovider", path.as_posix(), ] - runner, worker_output = _trusted_collection_runner( - controllers, root, pytest_args, collection_output - ) - command = [sys.executable, str(runner)] + worker = _trusted_collection_runner(controllers, root, pytest_args) + command = [sys.executable, str(worker)] digest = hashlib.sha256( json.dumps( { @@ -930,7 +906,6 @@ def _collect_node_ids( result, surviving = _managed_subprocess( command, cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), - writable_files=(collection_output, worker_output), readable_roots=(root, _CHECKER_PYTEST_PROBE), ) if result.returncode == 124: @@ -950,20 +925,17 @@ def _collect_node_ids( "collection left a surviving process-group descendant", ), (), ) - try: - payload = json.loads(collection_output.read_text(encoding="utf-8")) - if not isinstance(payload, list) or not all( - isinstance(item, str) for item in payload - ): - raise ValueError("node ID payload is malformed") - node_ids = tuple(sorted(payload)) - except (OSError, ValueError, json.JSONDecodeError): + node_ids = tuple(sorted( + line for line in result.stdout.splitlines() + if "::" in line and not line.startswith("=") + )) + if not node_ids and result.returncode == 0: return ( RunnerExecution( path.as_posix(), EvidenceOutcome.COLLECTION_ERROR, digest, - "protected collection probe produced no valid node IDs: " + "protected collection produced no valid node IDs: " + (result.stderr or result.stdout)[-500:], ), (), diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index ac522492a..72dd08619 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import shutil import signal import subprocess import sys @@ -28,25 +29,29 @@ def descendants(root): for pid in children.get(pending.pop(), ()): if pid not in found: found.add(pid); pending.append(pid) return found +def cleanup(): + deadline = time.monotonic() + .4 + while time.monotonic() < deadline: + pids = descendants(os.getpid()) + if child is not None and child.poll() is None: pids.add(child.pid) + if not pids: return + for pid in pids: + try: os.kill(pid, signal.SIGKILL) + except ProcessLookupError: pass + time.sleep(.01) def stop(_signum, _frame): - if child is not None: - deadline = time.monotonic() + .4 - while time.monotonic() < deadline: - pids = descendants(os.getpid()) | {child.pid} - if not pids: break - for pid in pids: - try: os.kill(pid, signal.SIGKILL) - except ProcessLookupError: pass - time.sleep(.01) - try: child.wait(timeout=.1) - except (subprocess.TimeoutExpired, ChildProcessError): pass + cleanup() raise SystemExit(124) signal.signal(signal.SIGTERM, stop) child = subprocess.Popen(sys.argv[1:], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) -stdout, stderr = child.communicate(sys.stdin.buffer.read()) -sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) -raise SystemExit(child.returncode) +try: + stdout, stderr = child.communicate(sys.stdin.buffer.read()) + sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) + status = child.returncode +finally: + cleanup() +raise SystemExit(status) """ @@ -66,17 +71,16 @@ def _marked_processes(token: str) -> set[int]: if marker in values: found.add(int(entry.name)) return found - result = subprocess.run( - ["ps", "eww", "-axo", "pid=,command="], - capture_output=True, text=True, check=False, - ) + result = subprocess.run(["ps", "eww", "-axo", "pid=,command="], + capture_output=True, text=True, check=False) text_marker = marker.decode() for line in result.stdout.splitlines(): - if text_marker not in line: + raw_pid, _separator, command_line = line.strip().partition(" ") + if text_marker not in command_line: continue try: - found.add(int(line.strip().split(None, 1)[0])) - except (ValueError, IndexError): + found.add(int(raw_pid)) + except ValueError: continue return found @@ -100,14 +104,41 @@ def _terminate_marked(token: str, leader: int, timeout: float = 0.5) -> None: pass +def _linux_contained_command(command: tuple[str, ...]) -> tuple[str, ...]: + """Place the signer behind a PID namespace whose init owns all descendants.""" + bwrap = shutil.which("bwrap") + if bwrap is None: + raise RuntimeError("protected Linux signer requires bubblewrap PID containment") + return ( + bwrap, "--unshare-pid", "--die-with-parent", "--new-session", + "--ro-bind", "/", "/", "--proc", "/proc", "--dev", "/dev", + "--tmpfs", "/tmp", "--", sys.executable, "-c", _LINUX_CONTAINMENT, + *command, + ) + + +def _kill_bounded(process: subprocess.Popen[bytes], token: str) -> None: + """Hard-kill the containment leader and inherited escapees within a bound.""" + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + _terminate_marked(token, process.pid) + try: + process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=0.5) + + def run_signer( command: tuple[str, ...], payload: bytes, *, timeout: float ) -> subprocess.CompletedProcess[bytes]: """Run a signer in a new process group and reap the complete group on timeout.""" token = uuid.uuid4().hex - contained_command = command - if sys.platform.startswith("linux"): - contained_command = (sys.executable, "-c", _LINUX_CONTAINMENT, *command) + contained_command = ( + _linux_contained_command(command) if sys.platform.startswith("linux") else command + ) process = subprocess.Popen( # pylint: disable=consider-using-with contained_command, stdin=subprocess.PIPE, @@ -121,20 +152,13 @@ def run_signer( except subprocess.TimeoutExpired as exc: if sys.platform.startswith("linux"): process.terminate() - else: try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - _terminate_marked(token, process.pid) - try: - stdout, stderr = process.communicate(timeout=0.5) - except subprocess.TimeoutExpired: - if process.stdout is not None: - process.stdout.close() - if process.stderr is not None: - process.stderr.close() - process.wait(timeout=0.5) + stdout, stderr = process.communicate(timeout=0.4) + except subprocess.TimeoutExpired: + _kill_bounded(process, token) + stdout, stderr = b"", b"" + else: + _kill_bounded(process, token) stdout, stderr = b"", b"" raise subprocess.TimeoutExpired( command, timeout, output=stdout, stderr=stderr diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index ab53260ea..d5991d993 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -15,6 +15,7 @@ import uuid from dataclasses import dataclass from pathlib import Path +import sysconfig @dataclass(frozen=True) @@ -28,15 +29,89 @@ class SupervisorLimits: max_processes: int = 128 +def _linked_libraries(path: Path) -> tuple[Path, ...]: + """Resolve the ELF loader closure for one executable or extension.""" + if not sys.platform.startswith("linux"): + return () + result = subprocess.run( + ["ldd", str(path)], capture_output=True, text=True, check=False + ) + libraries: set[Path] = set() + for line in result.stdout.splitlines(): + fields = line.strip().split() + candidates = ( + fields[2:3] if len(fields) >= 3 and fields[1:2] == ["=>"] else fields[:1] + ) + for value in candidates: + candidate = Path(value) + if candidate.is_absolute() and candidate.is_file(): + libraries.add(candidate.resolve()) + return tuple(sorted(libraries)) + + +def _runtime_directories() -> tuple[tuple[str, Path], ...]: + """Return Python directories whose complete readable contents are mounted.""" + locations = sysconfig.get_paths() + labels = { + "stdlib": locations.get("stdlib"), + "platstdlib": locations.get("platstdlib"), + "purelib": locations.get("purelib"), + "platlib": locations.get("platlib"), + } + seen: set[Path] = set() + result = [] + for label, value in labels.items(): + if not value: + continue + path = Path(value).resolve() + if path.is_dir() and path not in seen: + seen.add(path) + result.append((f"python-runtime/{label}", path)) + return tuple(result) + + +def released_runtime_closure_paths() -> tuple[tuple[str, Path], ...]: + """Return every regular file exposed by the sandbox with logical names.""" + entries: dict[str, Path] = {} + native: set[Path] = {Path(sys.executable).resolve()} + for label, directory in _runtime_directories(): + for path in sorted(directory.rglob("*")): + if path.is_file() and not path.is_symlink(): + entries[f"{label}/{path.relative_to(directory).as_posix()}"] = path + if path.suffix in {".so", ".dylib"}: + native.add(path) + sandbox_commands = { + "bwrap": shutil.which("bwrap"), + "setpriv": shutil.which("setpriv"), + "sudo": shutil.which("sudo"), + "mount": shutil.which("mount"), + "umount": shutil.which("umount"), + } + for name, value in sandbox_commands.items(): + if value: + path = Path(value).resolve() + entries[f"sandbox/{name}"] = path + native.add(path) + entries["interpreter/python"] = Path(sys.executable).resolve() + for path in sorted(native): + for library in _linked_libraries(path): + entries.setdefault(f"native/{library.name}", library) + return tuple(sorted(entries.items())) + + def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: - """Return the minimal host trees needed to start the configured interpreter.""" - roots = {cwd.resolve(), Path(sys.prefix)} - executable = shutil.which(command[0]) or command[0] - roots.add(Path(executable).parent) - for candidate in ("/bin", "/usr", "/lib", "/lib64"): - path = Path(candidate) - if path.exists(): - roots.add(path) + """Return the measured runtime closure plus the checker working directory.""" + roots: set[Path] = {cwd.resolve()} + directories = tuple(directory for _label, directory in _runtime_directories()) + roots.update(directories) + executable = Path(shutil.which(command[0]) or command[0]).resolve() + roots.add(executable) + for _label, path in released_runtime_closure_paths(): + if path.name in {"bwrap", "sudo", "mount", "umount"} or any( + path.is_relative_to(directory) for directory in directories + ): + continue + roots.add(path) return tuple(sorted(roots, key=lambda item: (len(item.parts), str(item)))) diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 6a21b920b..d297ff90e 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -3,6 +3,7 @@ import base64 import json import os +import shutil from pathlib import Path import subprocess import sys @@ -200,7 +201,7 @@ def short_run(command_value, payload, *, timeout): ) @pytest.mark.parametrize("adapter", ["attestation", "certificate"]) def test_remote_signer_normal_return_reaps_detached_descendant( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, adapter: str + tmp_path: Path, adapter: str ) -> None: marker = tmp_path / f"{adapter}.normal-return-leak" child = ( From 589c9b2c4280c82b425e5fcf5117f6b892b93cf2 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:20:45 -0700 Subject: [PATCH 091/128] fix(sync): mount post-drop sandbox executable --- pdd/sync_core/supervisor.py | 7 ++++++- tests/test_sync_core_runner.py | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index d5991d993..3a23fd251 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -107,7 +107,7 @@ def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: executable = Path(shutil.which(command[0]) or command[0]).resolve() roots.add(executable) for _label, path in released_runtime_closure_paths(): - if path.name in {"bwrap", "sudo", "mount", "umount"} or any( + if path.name in {"bwrap", "setpriv", "sudo", "mount", "umount"} or any( path.is_relative_to(directory) for directory in directories ): continue @@ -290,6 +290,11 @@ def bind(option: str, source: Path) -> None: argv.extend((option, f"@FD:{len(sources) - 1}@", str(source))) for item in _runtime_roots(command, workdir): bind("--ro-bind", item) + # ``setpriv`` executes after the namespace root is installed, so bind + # it and its ELF closure directly even when PATH resolution differs. + if setpriv is not None: + for item in (Path(setpriv).resolve(), *_linked_libraries(Path(setpriv))): + bind("--ro-bind", item) for item in readable_roots: bind("--ro-bind", item.resolve()) argv.extend(("--dev", "/dev")) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index fa32e9875..4b05e3904 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -903,8 +903,9 @@ def test_candidate_reading_worker_source_cannot_forge_collection_or_execution( "source = Path(sys.argv[0]).read_text(encoding='utf-8')\n" "for name in re.findall(r'/[^\\\" ]*/results-[0-9a-f]+/" "(?:collection|execution)-[0-9a-f]+\\.(?:json|xml)', source):\n" - " payload = ('[\\\"tests/test_widget.py::test_widget\\\"]' if name.endswith('.json') " - "else '')\n" + " payload = ('[\\\"tests/test_widget.py::test_widget\\\"]'\n" + " if name.endswith('.json') else\n" + " '')\n" " Path(name).write_text(payload, encoding='utf-8')\n" "if Path(sys.argv[0]).name in {'collection_worker.py', 'execution_worker.py'}:\n" " os._exit(80)\n" From 6aa15cfc48aaa1d4a92aaf9cb012607d30d57d5d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:24:36 -0700 Subject: [PATCH 092/128] fix(sync): preserve sandbox loader mount paths --- pdd/sync_core/supervisor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 3a23fd251..a694404e3 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -45,7 +45,10 @@ def _linked_libraries(path: Path) -> tuple[Path, ...]: for value in candidates: candidate = Path(value) if candidate.is_absolute() and candidate.is_file(): - libraries.add(candidate.resolve()) + # Keep the loader-visible spelling. Resolving /lib -> /usr/lib + # changes the destination inside the tmpfs namespace and makes + # an otherwise mounted ELF interpreter appear missing. + libraries.add(candidate) return tuple(sorted(libraries)) From b3dd5cfbe88982173ec42ef377b3c33d9ddf3ed3 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:28:01 -0700 Subject: [PATCH 093/128] fix(sync): tolerate unsupported host tool permissions --- pdd/sync_core/runner.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 0256f63a1..a6aa90ba4 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -636,9 +636,17 @@ def _released_runtime_closure_digest() -> str: """Hash the released runtime by logical name, never installation prefix.""" digest = hashlib.sha256() for name, path in _released_runtime_closure_paths(): - if not path.is_file() or path.is_symlink(): + if not path.is_file(): raise RuntimeError(f"released runtime entry is not a regular file: {name}") - digest.update(name.encode("utf-8") + b"\0" + path.read_bytes() + b"\0") + try: + content = path.read_bytes() + except PermissionError: + if sys.platform.startswith("linux"): + raise RuntimeError(f"released runtime entry is unreadable: {name}") from None + # macOS cannot execute the protected sandbox; do not make reporting + # unusable merely because a host-only outer helper is protected. + continue + digest.update(name.encode("utf-8") + b"\0" + content + b"\0") return digest.hexdigest() From ba58dff4a77979548a799d9093304eca02aa618e Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:29:41 -0700 Subject: [PATCH 094/128] test(sync): gate signer containment on Linux --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ed283ba28..e2e9cef98 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -95,6 +95,7 @@ jobs: pytest -q tests/test_sync_core_supervisor.py tests/test_sync_core_runner.py + tests/test_sync_core_trust.py tests/test_sync_core_lifecycle_scenarios.py tests/test_sync_core_reporting.py --timeout=60 From 05ca6ff784895391dea95d752d852ff418dfc1c1 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:33:09 -0700 Subject: [PATCH 095/128] fix(sync): preserve interpreter mount spelling --- pdd/sync_core/supervisor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index a694404e3..3429fd1af 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -107,8 +107,9 @@ def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: roots: set[Path] = {cwd.resolve()} directories = tuple(directory for _label, directory in _runtime_directories()) roots.update(directories) - executable = Path(shutil.which(command[0]) or command[0]).resolve() + executable = Path(shutil.which(command[0]) or command[0]) roots.add(executable) + roots.add(executable.resolve()) for _label, path in released_runtime_closure_paths(): if path.name in {"bwrap", "setpriv", "sudo", "mount", "umount"} or any( path.is_relative_to(directory) for directory in directories From c8af86997ff717a27a42a1eb28a57210be5dc468 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:42:48 -0700 Subject: [PATCH 096/128] perf(sync): cache measured runtime closure digest --- pdd/sync_core/runner.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index a6aa90ba4..66d499dc7 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -52,6 +52,7 @@ "--strict-config", "--strict-markers", "-ra", "-p", "no:cacheprovider" ) _CHECKER_PYTEST_PROBE = Path(__file__).with_name("pytest_probe.py").resolve() +_RUNTIME_DIGEST_CACHE: tuple[tuple[tuple[str, str, int, int], ...], str] | None = None @dataclass(frozen=True) @@ -634,8 +635,16 @@ def _released_runtime_closure_paths() -> tuple[tuple[str, Path], ...]: def _released_runtime_closure_digest() -> str: """Hash the released runtime by logical name, never installation prefix.""" + global _RUNTIME_DIGEST_CACHE + entries = _released_runtime_closure_paths() + manifest = tuple( + (name, str(path), path.stat().st_mtime_ns, path.stat().st_size) + for name, path in entries + ) + if _RUNTIME_DIGEST_CACHE is not None and _RUNTIME_DIGEST_CACHE[0] == manifest: + return _RUNTIME_DIGEST_CACHE[1] digest = hashlib.sha256() - for name, path in _released_runtime_closure_paths(): + for name, path in entries: if not path.is_file(): raise RuntimeError(f"released runtime entry is not a regular file: {name}") try: @@ -647,7 +656,9 @@ def _released_runtime_closure_digest() -> str: # unusable merely because a host-only outer helper is protected. continue digest.update(name.encode("utf-8") + b"\0" + content + b"\0") - return digest.hexdigest() + result = digest.hexdigest() + _RUNTIME_DIGEST_CACHE = manifest, result + return result def _git_paths(root: Path, ref: str) -> tuple[PurePosixPath, ...]: From b9167af0462260e41775d30a78a408c222c9e866 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:47:46 -0700 Subject: [PATCH 097/128] perf(sync): retain runtime byte-change invalidation --- pdd/sync_core/runner.py | 34 +++++++++++++++++++++++++--------- tests/test_sync_core_runner.py | 15 +++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 66d499dc7..d5eacc25c 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -52,7 +52,9 @@ "--strict-config", "--strict-markers", "-ra", "-p", "no:cacheprovider" ) _CHECKER_PYTEST_PROBE = Path(__file__).with_name("pytest_probe.py").resolve() -_RUNTIME_DIGEST_CACHE: tuple[tuple[tuple[str, str, int, int], ...], str] | None = None +_runtime_digest_cache: dict[ + str, tuple[tuple[tuple[str, Path], ...], tuple[tuple[str, str, int, int], ...], str] +] = {} @dataclass(frozen=True) @@ -633,16 +635,29 @@ def _released_runtime_closure_paths() -> tuple[tuple[str, Path], ...]: return tuple(sorted(paths, key=lambda item: item[0])) -def _released_runtime_closure_digest() -> str: - """Hash the released runtime by logical name, never installation prefix.""" - global _RUNTIME_DIGEST_CACHE - entries = _released_runtime_closure_paths() - manifest = tuple( +_default_runtime_closure_paths = _released_runtime_closure_paths + + +def _runtime_manifest( + entries: tuple[tuple[str, Path], ...] +) -> tuple[tuple[str, str, int, int], ...]: + """Return byte-change-sensitive metadata without rereading runtime bytes.""" + return tuple( (name, str(path), path.stat().st_mtime_ns, path.stat().st_size) for name, path in entries ) - if _RUNTIME_DIGEST_CACHE is not None and _RUNTIME_DIGEST_CACHE[0] == manifest: - return _RUNTIME_DIGEST_CACHE[1] + + +def _released_runtime_closure_digest() -> str: + """Hash the released runtime by logical name, never installation prefix.""" + default_paths = _released_runtime_closure_paths is _default_runtime_closure_paths + cached = _runtime_digest_cache.get("default") + if default_paths and cached is not None: + entries, cached_manifest, cached_digest = cached + if _runtime_manifest(entries) == cached_manifest: + return cached_digest + entries = _released_runtime_closure_paths() + manifest = _runtime_manifest(entries) digest = hashlib.sha256() for name, path in entries: if not path.is_file(): @@ -657,7 +672,8 @@ def _released_runtime_closure_digest() -> str: continue digest.update(name.encode("utf-8") + b"\0" + content + b"\0") result = digest.hexdigest() - _RUNTIME_DIGEST_CACHE = manifest, result + if default_paths: + _runtime_digest_cache["default"] = entries, manifest, result return result diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 4b05e3904..919bbcfd0 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -1057,6 +1057,21 @@ def test_released_runtime_digest_binds_installed_native_dependency( assert runner_module._released_runtime_closure_digest() != first +def test_default_runtime_digest_cache_invalidates_changed_native_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + native = tmp_path / "runtime" / "native_extension.so" + native.parent.mkdir() + native.write_bytes(b"native-v1") + provider = lambda: (("python-runtime/native_extension.so", native),) + monkeypatch.setattr(runner_module, "_released_runtime_closure_paths", provider) + monkeypatch.setattr(runner_module, "_default_runtime_closure_paths", provider) + monkeypatch.setattr(runner_module, "_runtime_digest_cache", {}) + first = runner_module._released_runtime_closure_digest() + native.write_bytes(b"native-v2") + assert runner_module._released_runtime_closure_digest() != first + + def test_released_runtime_digest_binds_runtime_and_sandbox_bytes_prefix_portably( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 00f93af334d18ba73cf541a3ff66846395f69927 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 09:50:42 -0700 Subject: [PATCH 098/128] perf(sync): cache released runtime path discovery --- pdd/sync_core/supervisor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 3429fd1af..edcbc9c5f 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -13,6 +13,7 @@ import threading import time import uuid +from functools import lru_cache from dataclasses import dataclass from pathlib import Path import sysconfig @@ -73,6 +74,7 @@ def _runtime_directories() -> tuple[tuple[str, Path], ...]: return tuple(result) +@lru_cache(maxsize=1) def released_runtime_closure_paths() -> tuple[tuple[str, Path], ...]: """Return every regular file exposed by the sandbox with logical names.""" entries: dict[str, Path] = {} From 369a676a874ddf956ff4b209ea2556eb66120d60 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:01:16 -0700 Subject: [PATCH 099/128] fix(sync): mount interpreter libraries and elevate signer scope --- pdd/sync_core/signer_process.py | 7 ++++++- pdd/sync_core/supervisor.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 72dd08619..c95708a58 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -109,7 +109,12 @@ def _linux_contained_command(command: tuple[str, ...]) -> tuple[str, ...]: bwrap = shutil.which("bwrap") if bwrap is None: raise RuntimeError("protected Linux signer requires bubblewrap PID containment") - return ( + sudo = shutil.which("sudo") + elevated = bool(sudo) and subprocess.run( + [sudo, "-n", "true"], capture_output=True, check=False + ).returncode == 0 + prefix = (sudo, "-n", "-E") if elevated and sudo is not None else () + return (*prefix, bwrap, "--unshare-pid", "--die-with-parent", "--new-session", "--ro-bind", "/", "/", "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--", sys.executable, "-c", _LINUX_CONTAINMENT, diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index edcbc9c5f..e2a9ac556 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -61,6 +61,7 @@ def _runtime_directories() -> tuple[tuple[str, Path], ...]: "platstdlib": locations.get("platstdlib"), "purelib": locations.get("purelib"), "platlib": locations.get("platlib"), + "prefix-lib": str(Path(sys.prefix) / "lib"), } seen: set[Path] = set() result = [] From dc2b651ed74ebed5031aeff7d0cf1fa07dbcf44d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:26:38 -0700 Subject: [PATCH 100/128] test(sync): cover resolved runtime library mounts --- tests/test_sync_core_supervisor.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index b33de80ea..21e476215 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -12,6 +12,7 @@ from pdd.sync_core.supervisor import ( SupervisorLimits, + _linked_libraries, _limited_command, _live_processes, _sandbox_command, @@ -50,6 +51,24 @@ def test_protected_runner_declares_finite_resource_limits() -> None: assert 0 < limits.max_processes <= 256 +def test_linked_libraries_keeps_loader_alias_and_resolved_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "libm-real.so" + target.write_bytes(b"library") + alias = tmp_path / "libm.so.6" + alias.symlink_to(target) + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr( + "pdd.sync_core.supervisor.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess( + [], 0, f"libm.so.6 => {alias} (0x0000)\n", "" + ), + ) + + assert _linked_libraries(tmp_path / "python") == (target, alias) + + @pytest.mark.skipif( not sys.platform.startswith("linux") or not shutil.which("bwrap"), reason="requires Linux kernel namespace containment", From a08e6d3a379f3d84e1d35b7b6d0506681c018051 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:27:08 -0700 Subject: [PATCH 101/128] fix(sync): bind resolved runtime dependencies --- pdd/sync_core/signer_process.py | 68 ++++++++++++++++++++++----------- pdd/sync_core/supervisor.py | 14 ++++--- 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index c95708a58..3907f1061 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -7,13 +7,14 @@ import signal import subprocess import sys +import tempfile import time import uuid from pathlib import Path _LINUX_CONTAINMENT = r""" -import ctypes, os, signal, subprocess, sys, time +import ctypes, os, pathlib, signal, subprocess, sys, time ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) child = None def descendants(root): @@ -43,8 +44,11 @@ def stop(_signum, _frame): cleanup() raise SystemExit(124) signal.signal(signal.SIGTERM, stop) +ready_path = os.environ.pop('PDD_SIGNER_READY_PATH', '') child = subprocess.Popen(sys.argv[1:], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) +if ready_path: + pathlib.Path(ready_path).touch(exist_ok=False) try: stdout, stderr = child.communicate(sys.stdin.buffer.read()) sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) @@ -136,6 +140,19 @@ def _kill_bounded(process: subprocess.Popen[bytes], token: str) -> None: process.wait(timeout=0.5) +def _wait_for_signer_start( + process: subprocess.Popen[bytes], ready_path: Path, command: tuple[str, ...], + timeout: float, token: str, +) -> None: + """Wait briefly for the containment init to launch the signer child.""" + deadline = time.monotonic() + 0.5 + while not ready_path.exists(): + if process.poll() is not None or time.monotonic() >= deadline: + _kill_bounded(process, token) + raise subprocess.TimeoutExpired(command, timeout) + time.sleep(0.005) + + def run_signer( command: tuple[str, ...], payload: bytes, *, timeout: float ) -> subprocess.CompletedProcess[bytes]: @@ -144,28 +161,35 @@ def run_signer( contained_command = ( _linux_contained_command(command) if sys.platform.startswith("linux") else command ) - process = subprocess.Popen( # pylint: disable=consider-using-with - contained_command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - start_new_session=True, - env=os.environ | {"PDD_SIGNER_PROCESS_TOKEN": token}, - ) - try: - stdout, stderr = process.communicate(payload, timeout=timeout) - except subprocess.TimeoutExpired as exc: + with tempfile.TemporaryDirectory(prefix="pdd-signer-") as directory: + ready_path = Path(directory) / "started" + environment = os.environ | {"PDD_SIGNER_PROCESS_TOKEN": token} if sys.platform.startswith("linux"): - process.terminate() - try: - stdout, stderr = process.communicate(timeout=0.4) - except subprocess.TimeoutExpired: + environment["PDD_SIGNER_READY_PATH"] = str(ready_path) + process = subprocess.Popen( # pylint: disable=consider-using-with + contained_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + env=environment, + ) + if sys.platform.startswith("linux"): + _wait_for_signer_start(process, ready_path, command, timeout, token) + try: + stdout, stderr = process.communicate(payload, timeout=timeout) + except subprocess.TimeoutExpired as exc: + if sys.platform.startswith("linux"): + process.terminate() + try: + stdout, stderr = process.communicate(timeout=0.4) + except subprocess.TimeoutExpired: + _kill_bounded(process, token) + stdout, stderr = b"", b"" + else: _kill_bounded(process, token) stdout, stderr = b"", b"" - else: - _kill_bounded(process, token) - stdout, stderr = b"", b"" - raise subprocess.TimeoutExpired( - command, timeout, output=stdout, stderr=stderr - ) from exc + raise subprocess.TimeoutExpired( + command, timeout, output=stdout, stderr=stderr + ) from exc return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index e2a9ac556..2f4d85a78 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -31,7 +31,7 @@ class SupervisorLimits: def _linked_libraries(path: Path) -> tuple[Path, ...]: - """Resolve the ELF loader closure for one executable or extension.""" + """Resolve loader-visible and physical paths for ELF dependencies.""" if not sys.platform.startswith("linux"): return () result = subprocess.run( @@ -46,10 +46,12 @@ def _linked_libraries(path: Path) -> tuple[Path, ...]: for value in candidates: candidate = Path(value) if candidate.is_absolute() and candidate.is_file(): - # Keep the loader-visible spelling. Resolving /lib -> /usr/lib - # changes the destination inside the tmpfs namespace and makes - # an otherwise mounted ELF interpreter appear missing. libraries.add(candidate) + # The dynamic loader may retain the /lib alias while the host + # mount operation follows it to /usr/lib. Bind both spellings + # so the empty namespace contains the loader's lookup path and + # the resolved dependency used by the host mount. + libraries.add(candidate.resolve()) return tuple(sorted(libraries)) @@ -101,7 +103,9 @@ def released_runtime_closure_paths() -> tuple[tuple[str, Path], ...]: entries["interpreter/python"] = Path(sys.executable).resolve() for path in sorted(native): for library in _linked_libraries(path): - entries.setdefault(f"native/{library.name}", library) + entries.setdefault( + f"native/{library.as_posix().lstrip('/')}", library + ) return tuple(sorted(entries.items())) From 9187877d8211e9acd83f90134819a890d33cf7b7 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:28:06 -0700 Subject: [PATCH 102/128] fix(sync): place signer readiness marker outside tmpfs --- pdd/sync_core/signer_process.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 3907f1061..127b1d46c 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -161,7 +161,9 @@ def run_signer( contained_command = ( _linux_contained_command(command) if sys.platform.startswith("linux") else command ) - with tempfile.TemporaryDirectory(prefix="pdd-signer-") as directory: + # The Linux signer scope replaces /tmp, so keep the checker-owned marker + # under its home directory, which the signer PID namespace bind-mounts. + with tempfile.TemporaryDirectory(prefix="pdd-signer-", dir=Path.home()) as directory: ready_path = Path(directory) / "started" environment = os.environ | {"PDD_SIGNER_PROCESS_TOKEN": token} if sys.platform.startswith("linux"): From 9b7e2950f525b7e75f845533c95c5f81f84c28ef Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:40:46 -0700 Subject: [PATCH 103/128] test(sync): cover measured loader search paths --- tests/test_sync_core_supervisor.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 21e476215..0da7fa81f 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -15,6 +15,7 @@ _linked_libraries, _limited_command, _live_processes, + _sandbox_library_path, _sandbox_command, run_supervised, ) @@ -69,6 +70,28 @@ def test_linked_libraries_keeps_loader_alias_and_resolved_path( assert _linked_libraries(tmp_path / "python") == (target, alias) +def test_sandbox_library_path_uses_only_measured_loader_directories( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + loader_alias = tmp_path / "lib" / "libm.so.6" + loader_target = tmp_path / "usr" / "lib" / "libm-real.so" + monkeypatch.setattr( + "pdd.sync_core.supervisor.released_runtime_closure_paths", + lambda: ( + ("native/lib/libm.so.6", loader_alias), + ("native/usr/lib/libm-real.so", loader_target), + ("interpreter/python", tmp_path / "python"), + ), + ) + + search_path = _sandbox_library_path({"LD_LIBRARY_PATH": "/checker/lib"}) + + assert search_path.split(os.pathsep)[:2] == [ + str(loader_alias.parent), str(loader_target.parent) + ] + assert "/checker/lib" in search_path.split(os.pathsep) + + @pytest.mark.skipif( not sys.platform.startswith("linux") or not shutil.which("bwrap"), reason="requires Linux kernel namespace containment", From c7a2f0aaad99e5fade1236a712459602431d9d2f Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:40:54 -0700 Subject: [PATCH 104/128] fix(sync): expose measured loader directories --- pdd/sync_core/supervisor.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 2f4d85a78..7aa2e4dfa 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -126,6 +126,20 @@ def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: return tuple(sorted(roots, key=lambda item: (len(item.parts), str(item)))) +def _sandbox_library_path(environment: dict[str, str]) -> str: + """Return loader search directories derived only from the measured closure.""" + directories = [] + for label, path in released_runtime_closure_paths(): + if label.startswith("native/"): + directories.append(str(path.parent)) + directories.append(str(Path(sys.prefix) / "lib")) + directories.extend( + item for item in environment.get("LD_LIBRARY_PATH", "").split(os.pathsep) + if item + ) + return os.pathsep.join(dict.fromkeys(directories)) + + def _limited_command(command: list[str], limits: SupervisorLimits) -> list[str]: """Apply non-raiseable POSIX limits after the namespace uid drop.""" script = ( @@ -341,13 +355,19 @@ def run_supervised( token = uuid.uuid4().hex stdout_file = tempfile.TemporaryFile(mode="w+b") stderr_file = tempfile.TemporaryFile(mode="w+b") + sandbox_environment = env | { + "PYTHONDONTWRITEBYTECODE": "1", + "PDD_SUPERVISION_TOKEN": token, + "TMPDIR": str(writable_roots[0].resolve()), + "TEMP": str(writable_roots[0].resolve()), + "TMP": str(writable_roots[0].resolve()), + } + library_path = _sandbox_library_path(env) + if library_path: + sandbox_environment["LD_LIBRARY_PATH"] = library_path process = subprocess.Popen( argv, cwd=cwd, stdout=stdout_file, stderr=stderr_file, - env=env | {"PYTHONDONTWRITEBYTECODE": "1", - "PDD_SUPERVISION_TOKEN": token, - "TMPDIR": str(writable_roots[0].resolve()), - "TEMP": str(writable_roots[0].resolve()), - "TMP": str(writable_roots[0].resolve())}, + env=sandbox_environment, start_new_session=True, ) timed_out = False From 2712920a7ed890c9e6b98f3cf6264aa7698519cd Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 10:44:10 -0700 Subject: [PATCH 105/128] test(sync): prevent mocked runtime cache poisoning --- .github/workflows/unit-tests.yml | 23 ++++++++++++++++++++++- tests/test_sync_core_supervisor.py | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e2e9cef98..046ea91f1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -65,12 +65,33 @@ jobs: mkdir -m 700 "$private_root/scratch" PRIVATE_ROOT="$private_root" python - <<'PY' import os + import json + import subprocess import sys from pathlib import Path - from pdd.sync_core.supervisor import run_supervised + from pdd.sync_core.supervisor import ( + _sandbox_command, + released_runtime_closure_paths, + run_supervised, + ) root = Path(os.environ["PRIVATE_ROOT"]) + resolved_interpreter = Path(sys.executable).resolve() + print("resolved interpreter:", resolved_interpreter) + print(subprocess.run( + ["ldd", str(resolved_interpreter)], check=False, text=True, + capture_output=True, + ).stdout) + sandbox, _ = _sandbox_command( + [sys.executable, "-c", "import math"], (root / "scratch",) + ) + print("sandbox destinations:", json.loads(sandbox[-2])) + print("runtime native paths:", [ + str(path) for label, path in released_runtime_closure_paths() + if label.startswith("native/") + ]) script = """from pathlib import Path + import math Path('ok').write_text('ok') try: Path('../outside').write_text('forbidden') diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 0da7fa81f..fff926e44 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -2,6 +2,7 @@ import os import json +import math import shutil import subprocess import sys @@ -30,6 +31,9 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( "pdd.sync_core.supervisor.subprocess.run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""), ) + monkeypatch.setattr( + "pdd.sync_core.supervisor.released_runtime_closure_paths", lambda: () + ) argv, profile = _sandbox_command(["/bin/true"], (tmp_path,)) assert profile is None assert argv[:3] == ["sudo", "-n", "-E"] @@ -52,6 +56,26 @@ def test_protected_runner_declares_finite_resource_limits() -> None: assert 0 < limits.max_processes <= 256 +@pytest.mark.skipif( + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux kernel namespace containment", +) +def test_sandboxed_python_imports_standard_library_after_command_construction( + tmp_path: Path, +) -> None: + result, surviving = run_supervised( + [sys.executable, "-c", "import math; print(math.pi)"], + cwd=tmp_path, + timeout=10, + env=dict(os.environ), + writable_roots=(tmp_path,), + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == str(math.pi) + assert surviving is False + + def test_linked_libraries_keeps_loader_alias_and_resolved_path( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From efb7f202e9104b57395e273bed921c7c44229b65 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 11:06:58 -0700 Subject: [PATCH 106/128] fix(sync): preserve protected execution diagnostics --- pdd/sync_core/runner.py | 28 ++++++++++------ pdd/sync_core/signer_process.py | 9 +++++- tests/test_sync_core_lifecycle_scenarios.py | 36 ++++++++++++++------- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index d5eacc25c..6fd7bd8fa 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -767,7 +767,7 @@ def _pytest_environment(home: Path) -> dict[str, str]: ) -def _trusted_probe_plugin(directory: Path) -> tuple[str, Path]: +def _trusted_probe_plugin(directory: Path, output: Path) -> tuple[str, Path]: """Create a unique plugin shim that loads the checker-owned probe by path.""" plugin_name = "pdd_checker_pytest_probe" plugin = directory / f"{plugin_name}.py" @@ -782,6 +782,7 @@ def _trusted_probe_plugin(directory: Path) -> tuple[str, Path]: " raise ImportError('checker pytest probe is unavailable')", "_MODULE = importlib.util.module_from_spec(_SPEC)", "_SPEC.loader.exec_module(_MODULE)", + f"_MODULE._OUTPUT_PATH = {json.dumps(str(output))}", "pytest_collection_modifyitems = _MODULE.pytest_collection_modifyitems", "", ) @@ -877,12 +878,15 @@ def _run_test_node( controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) - worker = _trusted_execution_runner(controllers, root, command[3:]) + junit = controllers / f"result-{os.urandom(16).hex()}.xml" + worker = _trusted_execution_runner( + controllers, root, [*command[3:], f"--junitxml={junit}"] + ) result, surviving = _managed_subprocess( [sys.executable, str(worker)], cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), - readable_roots=(root,), + writable_files=(junit,), readable_roots=(root,), ) if result.returncode == 124: return RunnerExecution( @@ -896,9 +900,8 @@ def _run_test_node( node_id, EvidenceOutcome.ERROR, command_digest, "validator left a surviving process-group descendant", ) - outcome, detail = _pytest_exit_outcome( - result.returncode, result.stdout + "\n" + result.stderr - ) + output = result.stdout + "\n" + result.stderr + outcome, detail = _junit_outcome(junit, result.returncode, output, 1) return RunnerExecution(node_id, outcome, command_digest, detail) @@ -915,6 +918,7 @@ def _collect_node_ids( controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) + probe_output = controllers / f"nodes-{os.urandom(16).hex()}.json" pytest_args = [ "--collect-only", "-q", @@ -924,6 +928,8 @@ def _collect_node_ids( "no:cacheprovider", path.as_posix(), ] + plugin_name, _plugin_directory = _trusted_probe_plugin(controllers, probe_output) + pytest_args.extend(("-p", plugin_name)) worker = _trusted_collection_runner(controllers, root, pytest_args) command = [sys.executable, str(worker)] digest = hashlib.sha256( @@ -941,6 +947,7 @@ def _collect_node_ids( result, surviving = _managed_subprocess( command, cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), + writable_files=(probe_output,), readable_roots=(root, _CHECKER_PYTEST_PROBE), ) if result.returncode == 124: @@ -960,10 +967,11 @@ def _collect_node_ids( "collection left a surviving process-group descendant", ), (), ) - node_ids = tuple(sorted( - line for line in result.stdout.splitlines() - if "::" in line and not line.startswith("=") - )) + try: + values = json.loads(probe_output.read_text(encoding="utf-8")) + node_ids = tuple(sorted(item for item in values if isinstance(item, str))) + except (OSError, json.JSONDecodeError): + node_ids = () if not node_ids and result.returncode == 0: return ( RunnerExecution( diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 127b1d46c..0e975c9f7 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -176,10 +176,17 @@ def run_signer( start_new_session=True, env=environment, ) + try: + if process.stdin is not None: + process.stdin.write(payload) + process.stdin.close() + process.stdin = None + except BrokenPipeError: + pass if sys.platform.startswith("linux"): _wait_for_signer_start(process, ready_path, command, timeout, token) try: - stdout, stderr = process.communicate(payload, timeout=timeout) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired as exc: if sys.platform.startswith("linux"): process.terminate() diff --git a/tests/test_sync_core_lifecycle_scenarios.py b/tests/test_sync_core_lifecycle_scenarios.py index 0762a3b69..f28d7a07c 100644 --- a/tests/test_sync_core_lifecycle_scenarios.py +++ b/tests/test_sync_core_lifecycle_scenarios.py @@ -11,6 +11,7 @@ from pathlib import Path, PurePosixPath import pytest +import pdd.sync_core.lifecycle as lifecycle_module from pdd.sync_core.certificate import LifecycleResult from pdd.sync_core.lifecycle import ( @@ -176,13 +177,11 @@ def test_lifecycle_measurement_rejects_synthesized_compatibility_fields() -> Non def test_lifecycle_commands_do_not_use_unsupervised_subprocess_run(monkeypatch) -> None: - from pdd.sync_core import lifecycle - def forbidden(*_args, **_kwargs): pytest.fail("lifecycle command bypassed the shared sandbox supervisor") - monkeypatch.setattr(lifecycle.subprocess, "run", forbidden) - assert lifecycle._candidate_interpreter_identity( + monkeypatch.setattr(lifecycle_module.subprocess, "run", forbidden) + assert lifecycle_module._candidate_interpreter_identity( Path(sys.executable), {"PATH": os.environ.get("PATH", "")} ) is not None @@ -355,14 +354,29 @@ def test_candidate_install_e2e_uses_locked_runtime_wheelhouse(tmp_path) -> None: f"{hashlib.sha256(runtime.read_bytes()).hexdigest()}\n", encoding="utf-8", ) - installed = _install_candidate_wheel( - tmp_path, - tmp_path / "home", - candidate, - wheelhouse, - lock, + commands = [] + actual_command = lifecycle_module._lifecycle_command + + def traced_command(*args, **kwargs): + result = actual_command(*args, **kwargs) + commands.append(result) + return result + + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(lifecycle_module, "_lifecycle_command", traced_command) + try: + installed = _install_candidate_wheel( + tmp_path, + tmp_path / "home", + candidate, + wheelhouse, + lock, + ) + finally: + monkeypatch.undo() + assert installed is not None, "\n".join( + result.stderr for result in commands if result.stderr ) - assert installed is not None candidate_python, dependency_digest = installed pyvenv = candidate_python.parents[1] / "pyvenv.cfg" assert "include-system-site-packages = false" in pyvenv.read_text(encoding="utf-8") From 5b34e3b9f326d5c7ae4c3b908cf3bf78a750bdb8 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 11:21:40 -0700 Subject: [PATCH 107/128] fix(sync): retain checker diagnostics outside candidate files --- pdd/sync_core/pytest_probe.py | 2 ++ pdd/sync_core/runner.py | 20 ++++++++++---------- pdd/sync_core/supervisor.py | 5 +++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pdd/sync_core/pytest_probe.py b/pdd/sync_core/pytest_probe.py index 665f49879..5ade4ba05 100644 --- a/pdd/sync_core/pytest_probe.py +++ b/pdd/sync_core/pytest_probe.py @@ -24,3 +24,5 @@ def pytest_collection_modifyitems(items): json.dumps(protected_node_ids, separators=(",", ":")), encoding="utf-8", ) + else: + print("PDD_PROTECTED_NODE_IDS=" + json.dumps(protected_node_ids)) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 6fd7bd8fa..dd373eb92 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -767,7 +767,7 @@ def _pytest_environment(home: Path) -> dict[str, str]: ) -def _trusted_probe_plugin(directory: Path, output: Path) -> tuple[str, Path]: +def _trusted_probe_plugin(directory: Path) -> tuple[str, Path]: """Create a unique plugin shim that loads the checker-owned probe by path.""" plugin_name = "pdd_checker_pytest_probe" plugin = directory / f"{plugin_name}.py" @@ -782,7 +782,6 @@ def _trusted_probe_plugin(directory: Path, output: Path) -> tuple[str, Path]: " raise ImportError('checker pytest probe is unavailable')", "_MODULE = importlib.util.module_from_spec(_SPEC)", "_SPEC.loader.exec_module(_MODULE)", - f"_MODULE._OUTPUT_PATH = {json.dumps(str(output))}", "pytest_collection_modifyitems = _MODULE.pytest_collection_modifyitems", "", ) @@ -918,7 +917,6 @@ def _collect_node_ids( controllers.mkdir(mode=0o700) home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) - probe_output = controllers / f"nodes-{os.urandom(16).hex()}.json" pytest_args = [ "--collect-only", "-q", @@ -928,7 +926,7 @@ def _collect_node_ids( "no:cacheprovider", path.as_posix(), ] - plugin_name, _plugin_directory = _trusted_probe_plugin(controllers, probe_output) + plugin_name, _plugin_directory = _trusted_probe_plugin(controllers) pytest_args.extend(("-p", plugin_name)) worker = _trusted_collection_runner(controllers, root, pytest_args) command = [sys.executable, str(worker)] @@ -947,7 +945,6 @@ def _collect_node_ids( result, surviving = _managed_subprocess( command, cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), - writable_files=(probe_output,), readable_roots=(root, _CHECKER_PYTEST_PROBE), ) if result.returncode == 124: @@ -967,11 +964,14 @@ def _collect_node_ids( "collection left a surviving process-group descendant", ), (), ) - try: - values = json.loads(probe_output.read_text(encoding="utf-8")) - node_ids = tuple(sorted(item for item in values if isinstance(item, str))) - except (OSError, json.JSONDecodeError): - node_ids = () + node_ids = () + for line in result.stdout.splitlines(): + if line.startswith("PDD_PROTECTED_NODE_IDS="): + try: + values = json.loads(line.partition("=")[2]) + node_ids = tuple(sorted(item for item in values if isinstance(item, str))) + except json.JSONDecodeError: + pass if not node_ids and result.returncode == 0: return ( RunnerExecution( diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 7aa2e4dfa..e83e04148 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -115,8 +115,9 @@ def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: directories = tuple(directory for _label, directory in _runtime_directories()) roots.update(directories) executable = Path(shutil.which(command[0]) or command[0]) - roots.add(executable) - roots.add(executable.resolve()) + if not executable.is_relative_to(cwd): + roots.add(executable) + roots.add(executable.resolve()) for _label, path in released_runtime_closure_paths(): if path.name in {"bwrap", "setpriv", "sudo", "mount", "umount"} or any( path.is_relative_to(directory) for directory in directories From fb82bc416ffa03011e2fa77a94f9ea3ad369ee9a Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 11:28:33 -0700 Subject: [PATCH 108/128] fix(sync): retain bound directory ancestry --- pdd/sync_core/supervisor.py | 2 ++ tests/test_sync_core_supervisor.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index e83e04148..20acfdc10 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -314,6 +314,8 @@ def bind(option: str, source: Path) -> None: destination_dirs.add(directory) sources.append(source) argv.extend((option, f"@FD:{len(sources) - 1}@", str(source))) + if source.is_dir(): + destination_dirs.add(source) for item in _runtime_roots(command, workdir): bind("--ro-bind", item) # ``setpriv`` executes after the namespace root is installed, so bind diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index fff926e44..cfa6edb20 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -47,6 +47,32 @@ def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( assert bwrap[bwrap.index("--ro-bind") + 1].startswith("@FD:") +def test_sandbox_directory_bind_provides_parent_for_nested_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + nested = tmp_path / "candidate-venv" / "bin" + nested.mkdir(parents=True) + interpreter = nested / "python" + interpreter.write_text("python", encoding="utf-8") + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr( + "pdd.sync_core.supervisor.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""), + ) + monkeypatch.setattr( + "pdd.sync_core.supervisor.released_runtime_closure_paths", lambda: () + ) + + argv, _profile = _sandbox_command([str(interpreter)], (tmp_path,), cwd=tmp_path) + bwrap = json.loads(argv[-2]) + directory_targets = { + bwrap[index + 1] for index, value in enumerate(bwrap[:-1]) + if value == "--dir" + } + assert str(nested) not in directory_targets + + def test_protected_runner_declares_finite_resource_limits() -> None: limits = SupervisorLimits() assert 0 < limits.max_output_bytes <= 16 * 1024 * 1024 From 8700a6b60518399b8d9dc7d28e7780550cfe7ca2 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 11:41:42 -0700 Subject: [PATCH 109/128] fix(sync): bind resolved nested interpreters --- pdd/sync_core/supervisor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 20acfdc10..07ab58c15 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -115,9 +115,11 @@ def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: directories = tuple(directory for _label, directory in _runtime_directories()) roots.update(directories) executable = Path(shutil.which(command[0]) or command[0]) + resolved_executable = executable.resolve() if not executable.is_relative_to(cwd): roots.add(executable) - roots.add(executable.resolve()) + if not resolved_executable.is_relative_to(cwd): + roots.add(resolved_executable) for _label, path in released_runtime_closure_paths(): if path.name in {"bwrap", "setpriv", "sudo", "mount", "umount"} or any( path.is_relative_to(directory) for directory in directories From 2963efaf91103fc211d1dae29c7068ec5af93318 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 11:57:51 -0700 Subject: [PATCH 110/128] test(sync): cover interpreter sandbox mount destinations --- tests/test_sync_core_supervisor.py | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index cfa6edb20..453635399 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -73,6 +73,83 @@ def test_sandbox_directory_bind_provides_parent_for_nested_file( assert str(nested) not in directory_targets +def test_sandbox_binds_resolved_runtime_sources_at_original_destinations( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The namespace must retain the executable and loader lookup spellings.""" + runtime = tmp_path / "runtime" + executable_source = runtime / "python-real" + executable_source.parent.mkdir() + executable_source.write_text("python", encoding="utf-8") + executable_destination = tmp_path / "toolcache" / "bin" / "python" + executable_destination.parent.mkdir(parents=True) + executable_destination.symlink_to(executable_source) + loader_source = runtime / "libc-real.so" + loader_source.write_bytes(b"library") + loader_destination = tmp_path / "loader" / "libc.so.6" + loader_destination.parent.mkdir() + loader_destination.symlink_to(loader_source) + + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(sys, "executable", str(executable_destination)) + monkeypatch.setattr( + shutil, + "which", + lambda name: {"bwrap": "/usr/bin/bwrap", "sudo": "/usr/bin/sudo", + "setpriv": "/usr/bin/setpriv"}.get(name), + ) + monkeypatch.setattr( + "pdd.sync_core.supervisor.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""), + ) + monkeypatch.setattr("pdd.sync_core.supervisor._runtime_directories", lambda: ()) + monkeypatch.setattr( + "pdd.sync_core.supervisor.released_runtime_closure_paths", + lambda: ( + ("native/loader/libc.so.6", loader_destination), + ("native/runtime/libc-real.so", loader_source), + ), + ) + + argv, _profile = _sandbox_command( + [str(executable_destination), "-c", "pass"], (tmp_path,), cwd=tmp_path + ) + bwrap = json.loads(argv[-2]) + sources = json.loads(argv[-1]) + + def bind_source(destination: Path) -> str: + index = bwrap.index(str(destination)) + assert bwrap[index - 2] == "--ro-bind" + placeholder = bwrap[index - 1] + assert placeholder.startswith("@FD:") and placeholder.endswith("@") + return sources[int(placeholder[4:-1])] + + assert str(executable_destination.parent) in { + bwrap[index + 1] for index, value in enumerate(bwrap[:-1]) + if value == "--dir" + } + assert bind_source(executable_destination) == str(executable_source) + assert bind_source(loader_destination) == str(loader_source) + + +@pytest.mark.skipif( + not sys.platform.startswith("linux") or not shutil.which("bwrap"), + reason="requires Linux kernel namespace containment", +) +def test_sandboxed_python_minimal_smoke(tmp_path: Path) -> None: + """A protected namespace can start the configured interpreter.""" + result, surviving = run_supervised( + [sys.executable, "-c", "pass"], + cwd=tmp_path, + timeout=10, + env=dict(os.environ), + writable_roots=(tmp_path,), + ) + + assert result.returncode == 0, result.stderr + assert surviving is False + + def test_protected_runner_declares_finite_resource_limits() -> None: limits = SupervisorLimits() assert 0 < limits.max_output_bytes <= 16 * 1024 * 1024 From 9f07c4e10d50f559bdf08dd67570e106a6b8c145 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 11:59:40 -0700 Subject: [PATCH 111/128] fix(sync): bind runtime paths at namespace spellings --- pdd/sync_core/supervisor.py | 31 ++++++++++++++++-------------- tests/test_sync_core_supervisor.py | 14 ++++++++++---- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 07ab58c15..74b54468f 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -291,13 +291,12 @@ def _sandbox_command( "unsupported protected sandbox: macOS cannot prove process lifetime isolation" ) if sys.platform.startswith("linux") and shutil.which("bwrap"): - elevated = bool(shutil.which("sudo")) and subprocess.run( - ["sudo", "-n", "true"], capture_output=True, check=False, - ).returncode == 0 - if not elevated: + if not (bool(shutil.which("sudo")) and subprocess.run( + ["sudo", "-n", "true"], capture_output=True, check=False, + ).returncode == 0): raise RuntimeError("protected sandbox requires privileged bind staging") - setpriv = shutil.which("setpriv") if elevated else None - if elevated and setpriv is None: + setpriv = shutil.which("setpriv") + if setpriv is None: raise RuntimeError("protected sandbox requires setpriv for post-mount uid drop") workdir = (cwd or Path.cwd()).resolve() argv = ["bwrap", "--unshare-ipc", "--unshare-pid", "--unshare-net", @@ -305,9 +304,10 @@ def _sandbox_command( "--tmpfs", "/", "--proc", "/proc", "--dir", "/tmp"] sources: list[Path] = [] destination_dirs = {Path("/tmp")} - def bind(option: str, source: Path) -> None: + def bind(option: str, source: Path, destination: Path | None = None) -> None: + destination = destination or source missing = [] - parent = source.parent + parent = destination.parent while parent != Path("/") and parent not in destination_dirs: missing.append(parent) parent = parent.parent @@ -315,16 +315,19 @@ def bind(option: str, source: Path) -> None: argv.extend(("--dir", str(directory))) destination_dirs.add(directory) sources.append(source) - argv.extend((option, f"@FD:{len(sources) - 1}@", str(source))) - if source.is_dir(): - destination_dirs.add(source) + argv.extend((option, f"@FD:{len(sources) - 1}@", str(destination))) + if destination.is_dir(): + destination_dirs.add(destination) for item in _runtime_roots(command, workdir): - bind("--ro-bind", item) + # A host bind follows symlinks, but the process command and ELF + # loader retain their original spellings in the new namespace. + bind("--ro-bind", item.resolve(), item) # ``setpriv`` executes after the namespace root is installed, so bind # it and its ELF closure directly even when PATH resolution differs. if setpriv is not None: - for item in (Path(setpriv).resolve(), *_linked_libraries(Path(setpriv))): - bind("--ro-bind", item) + setpriv_path = Path(setpriv) + for item in (setpriv_path, *_linked_libraries(setpriv_path)): + bind("--ro-bind", item.resolve(), item) for item in readable_roots: bind("--ro-bind", item.resolve()) argv.extend(("--dev", "/dev")) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 453635399..664e49e3c 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -89,20 +89,26 @@ def test_sandbox_binds_resolved_runtime_sources_at_original_destinations( loader_destination = tmp_path / "loader" / "libc.so.6" loader_destination.parent.mkdir() loader_destination.symlink_to(loader_source) + workdir = tmp_path / "work" + workdir.mkdir() monkeypatch.setattr(sys, "platform", "linux") monkeypatch.setattr(sys, "executable", str(executable_destination)) + sandbox_tools = { + "bwrap": "/usr/bin/bwrap", + "sudo": "/usr/bin/sudo", + "setpriv": "/usr/bin/setpriv", + } monkeypatch.setattr( shutil, "which", - lambda name: {"bwrap": "/usr/bin/bwrap", "sudo": "/usr/bin/sudo", - "setpriv": "/usr/bin/setpriv"}.get(name), + sandbox_tools.get, ) monkeypatch.setattr( "pdd.sync_core.supervisor.subprocess.run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, "", ""), ) - monkeypatch.setattr("pdd.sync_core.supervisor._runtime_directories", lambda: ()) + monkeypatch.setattr("pdd.sync_core.supervisor._runtime_directories", tuple) monkeypatch.setattr( "pdd.sync_core.supervisor.released_runtime_closure_paths", lambda: ( @@ -112,7 +118,7 @@ def test_sandbox_binds_resolved_runtime_sources_at_original_destinations( ) argv, _profile = _sandbox_command( - [str(executable_destination), "-c", "pass"], (tmp_path,), cwd=tmp_path + [str(executable_destination), "-c", "pass"], (workdir,), cwd=workdir ) bwrap = json.loads(argv[-2]) sources = json.loads(argv[-1]) From 89afb96dc246522be264cea4e2c86c89a41b8273 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:13:29 -0700 Subject: [PATCH 112/128] test(sync): cover limited command interpreter mount --- tests/test_sync_core_supervisor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index 664e49e3c..bf55e2737 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -89,6 +89,9 @@ def test_sandbox_binds_resolved_runtime_sources_at_original_destinations( loader_destination = tmp_path / "loader" / "libc.so.6" loader_destination.parent.mkdir() loader_destination.symlink_to(loader_source) + candidate = tmp_path / "candidate" / "bin" / "python" + candidate.parent.mkdir(parents=True) + candidate.write_text("python", encoding="utf-8") workdir = tmp_path / "work" workdir.mkdir() @@ -118,7 +121,7 @@ def test_sandbox_binds_resolved_runtime_sources_at_original_destinations( ) argv, _profile = _sandbox_command( - [str(executable_destination), "-c", "pass"], (workdir,), cwd=workdir + [str(candidate), "-c", "pass"], (workdir,), cwd=workdir ) bwrap = json.loads(argv[-2]) sources = json.loads(argv[-1]) From 50816e03b9acc70e179812d1862d34736a955dfd Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:14:39 -0700 Subject: [PATCH 113/128] fix(sync): mount limited command interpreter --- pdd/sync_core/supervisor.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index 74b54468f..b5330fd1d 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -114,12 +114,15 @@ def _runtime_roots(command: list[str], cwd: Path) -> tuple[Path, ...]: roots: set[Path] = {cwd.resolve()} directories = tuple(directory for _label, directory in _runtime_directories()) roots.update(directories) - executable = Path(shutil.which(command[0]) or command[0]) - resolved_executable = executable.resolve() - if not executable.is_relative_to(cwd): - roots.add(executable) - if not resolved_executable.is_relative_to(cwd): - roots.add(resolved_executable) + executables = ( + Path(sys.executable), Path(shutil.which(command[0]) or command[0]), + ) + for executable in executables: + resolved_executable = executable.resolve() + if not executable.is_relative_to(cwd): + roots.add(executable) + if not resolved_executable.is_relative_to(cwd): + roots.add(resolved_executable) for _label, path in released_runtime_closure_paths(): if path.name in {"bwrap", "setpriv", "sudo", "mount", "umount"} or any( path.is_relative_to(directory) for directory in directories From c15cbddbc59ce6718f1edfaa703d0a7498c8517e Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:27:27 -0700 Subject: [PATCH 114/128] test(sync): expose protected collection failures early --- .github/workflows/unit-tests.yml | 6 ++++++ pdd/sync_core/runner.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 046ea91f1..d9380d03e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -111,6 +111,12 @@ jobs: assert not (root / "outside").exists() PY + - name: Verify protected pytest smoke + run: > + pytest -q + tests/test_sync_core_runner.py::test_passing_collected_test_is_pass + --timeout=60 + - name: Run focused protected-runner tests run: > pytest -q diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index dd373eb92..47353a3ed 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -988,7 +988,10 @@ def _collect_node_ids( detail = "zero protected node IDs collected" elif result.returncode != 0: outcome = EvidenceOutcome.COLLECTION_ERROR + diagnostic = (result.stderr or result.stdout).strip()[-1000:] detail = "protected sandbox rejected pytest collection" + if diagnostic: + detail += f": {diagnostic}" else: outcome = EvidenceOutcome.PASS detail = f"{len(node_ids)} protected node IDs collected" From c016be9dc21547d85f9caf0373c973210c34190d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:36:17 -0700 Subject: [PATCH 115/128] perf(sync): hash measured runtime concurrently --- pdd/sync_core/runner.py | 39 +++++++++++++++++++++++----------- tests/test_sync_core_runner.py | 27 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 47353a3ed..b1bd33eb3 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -7,6 +7,7 @@ import hashlib import ast import configparser +import concurrent.futures import json import os import platform @@ -648,6 +649,25 @@ def _runtime_manifest( ) +def _hash_runtime_entry(entry: tuple[str, Path]) -> tuple[str, bytes] | None: + """Hash one measured runtime file without loading it wholly into memory.""" + name, path = entry + if not path.is_file(): + raise RuntimeError(f"released runtime entry is not a regular file: {name}") + file_digest = hashlib.sha256() + try: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + file_digest.update(chunk) + except PermissionError: + if sys.platform.startswith("linux"): + raise RuntimeError(f"released runtime entry is unreadable: {name}") from None + # macOS cannot execute the protected sandbox; do not make reporting + # unusable merely because a host-only outer helper is protected. + return None + return name, file_digest.digest() + + def _released_runtime_closure_digest() -> str: """Hash the released runtime by logical name, never installation prefix.""" default_paths = _released_runtime_closure_paths is _default_runtime_closure_paths @@ -658,19 +678,14 @@ def _released_runtime_closure_digest() -> str: return cached_digest entries = _released_runtime_closure_paths() manifest = _runtime_manifest(entries) + worker_count = min(32, (os.cpu_count() or 1) + 4) + with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as executor: + hashed_entries = tuple(executor.map(_hash_runtime_entry, entries)) digest = hashlib.sha256() - for name, path in entries: - if not path.is_file(): - raise RuntimeError(f"released runtime entry is not a regular file: {name}") - try: - content = path.read_bytes() - except PermissionError: - if sys.platform.startswith("linux"): - raise RuntimeError(f"released runtime entry is unreadable: {name}") from None - # macOS cannot execute the protected sandbox; do not make reporting - # unusable merely because a host-only outer helper is protected. - continue - digest.update(name.encode("utf-8") + b"\0" + content + b"\0") + for hashed_entry in hashed_entries: + if hashed_entry is not None: + name, content_digest = hashed_entry + digest.update(name.encode("utf-8") + b"\0" + content_digest + b"\0") result = digest.hexdigest() if default_paths: _runtime_digest_cache["default"] = entries, manifest, result diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 919bbcfd0..6482162ce 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -4,6 +4,7 @@ import shutil import subprocess import sys +import threading from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -1072,6 +1073,32 @@ def test_default_runtime_digest_cache_invalidates_changed_native_bytes( assert runner_module._released_runtime_closure_digest() != first +def test_released_runtime_digest_hashes_entries_concurrently( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + entries = [] + for index in range(8): + path = tmp_path / f"runtime-{index}" + path.write_bytes(str(index).encode()) + entries.append((path.name, path)) + barrier = threading.Barrier(2) + thread_ids = set() + original = runner_module._hash_runtime_entry + + def observed(entry): + thread_ids.add(threading.get_ident()) + if len(thread_ids) <= 2: + barrier.wait(timeout=2) + return original(entry) + + monkeypatch.setattr(runner_module, "_released_runtime_closure_paths", lambda: tuple(entries)) + monkeypatch.setattr(runner_module, "_hash_runtime_entry", observed) + + runner_module._released_runtime_closure_digest() + + assert len(thread_ids) > 1 + + def test_released_runtime_digest_binds_runtime_and_sandbox_bytes_prefix_portably( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 204d3851f65dacba6b04119d0307c3a9a4d3aeab Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:41:55 -0700 Subject: [PATCH 116/128] fix(sync): expose trusted collection probe --- pdd/sync_core/runner.py | 9 +++++---- tests/test_sync_core_runner.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index b1bd33eb3..9b45fc55d 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -824,12 +824,13 @@ def _trusted_collection_runner( "import os, subprocess, sys", "", f"_ROOT = {json.dumps(str(root))}", + f"_CONTROLLER = {json.dumps(str(directory))}", "", "os.chdir(_ROOT)", - "if _ROOT not in sys.path:", - " sys.path.insert(0, _ROOT)", - "_STATUS = subprocess.run([sys.executable, '-m', 'pytest'] + " - + json.dumps(pytest_args) + ").returncode", + "_ENV = os.environ.copy()", + "_ENV['PYTHONPATH'] = _CONTROLLER", + "_STATUS = subprocess.run([sys.executable, '-P', '-m', 'pytest'] + " + + json.dumps(pytest_args) + ", env=_ENV).returncode", "sys.stdout.flush(); sys.stderr.flush()", "os._exit(_STATUS if _STATUS in (0, 5) else 125)", "", diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 6482162ce..be3925e8e 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -616,6 +616,21 @@ def test_collection_probe_fixed_name_is_not_candidate_shadowable(tmp_path) -> No assert not (root / "candidate-fixed-probe-loaded").exists() +def test_collection_worker_uses_trusted_plugin_path(tmp_path: Path) -> None: + root = tmp_path / "candidate" + controller = tmp_path / "controller" + root.mkdir() + controller.mkdir() + + worker = runner_module._trusted_collection_runner(controller, root, ["tests"]) + source = worker.read_text(encoding="utf-8") + + assert "_CONTROLLER =" in source + assert str(controller) in source + assert "_ENV['PYTHONPATH'] = _CONTROLLER" in source + assert "[sys.executable, '-P', '-m', 'pytest']" in source + + def test_deselected_declared_test_cannot_pass(tmp_path) -> None: content = "def test_keep(): assert True\ndef test_drop(): assert True\n" root, head = _repository(tmp_path, content) From 8e3c86306fad25270c5d8de0bd7cb8edf7e68023 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:47:42 -0700 Subject: [PATCH 117/128] fix(sync): create trusted junit channel before binding --- pdd/sync_core/runner.py | 1 + tests/test_sync_core_runner.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 9b45fc55d..599cc333d 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -894,6 +894,7 @@ def _run_test_node( home = temporary / "scratch" / "home" home.mkdir(mode=0o700, parents=True) junit = controllers / f"result-{os.urandom(16).hex()}.xml" + junit.touch(mode=0o600) worker = _trusted_execution_runner( controllers, root, [*command[3:], f"--junitxml={junit}"] ) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index be3925e8e..2e9a0aadc 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -631,6 +631,27 @@ def test_collection_worker_uses_trusted_plugin_path(tmp_path: Path) -> None: assert "[sys.executable, '-P', '-m', 'pytest']" in source +def test_execution_precreates_private_junit_channel( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def supervised(command, **kwargs): + del command + junit, = kwargs["writable_files"] + assert junit.is_file() + assert junit.stat().st_mode & 0o777 == 0o600 + junit.write_text( + '', + encoding="utf-8", + ) + return subprocess.CompletedProcess([], 0, "", ""), False + + monkeypatch.setattr(runner_module, "_managed_subprocess", supervised) + + execution = runner_module._run_test_node(tmp_path, "test_widget.py::test_widget", 10) + + assert execution.outcome is EvidenceOutcome.PASS + + def test_deselected_declared_test_cannot_pass(tmp_path) -> None: content = "def test_keep(): assert True\ndef test_drop(): assert True\n" root, head = _repository(tmp_path, content) From d03115bfc42c13915ad865adb3d1c8de34ad6362 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 12:54:08 -0700 Subject: [PATCH 118/128] perf(sync): deduplicate measured runtime roots --- pdd/sync_core/supervisor.py | 17 +++++++++++------ tests/test_sync_core_supervisor.py | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/pdd/sync_core/supervisor.py b/pdd/sync_core/supervisor.py index b5330fd1d..3fa340e92 100644 --- a/pdd/sync_core/supervisor.py +++ b/pdd/sync_core/supervisor.py @@ -63,17 +63,22 @@ def _runtime_directories() -> tuple[tuple[str, Path], ...]: "platstdlib": locations.get("platstdlib"), "purelib": locations.get("purelib"), "platlib": locations.get("platlib"), - "prefix-lib": str(Path(sys.prefix) / "lib"), } - seen: set[Path] = set() - result = [] + candidates = [] for label, value in labels.items(): if not value: continue path = Path(value).resolve() - if path.is_dir() and path not in seen: - seen.add(path) - result.append((f"python-runtime/{label}", path)) + if path.is_dir() and path not in {item[1] for item in candidates}: + candidates.append((label, path)) + result = [] + for label, path in sorted(candidates, key=lambda item: len(item[1].parts)): + if any(path.is_relative_to(parent) for _parent_label, parent in result): + continue + result.append((label, path)) + result = [ + (f"python-runtime/{label}", path) for label, path in result + ] return tuple(result) diff --git a/tests/test_sync_core_supervisor.py b/tests/test_sync_core_supervisor.py index bf55e2737..3d76375d4 100644 --- a/tests/test_sync_core_supervisor.py +++ b/tests/test_sync_core_supervisor.py @@ -18,10 +18,35 @@ _live_processes, _sandbox_library_path, _sandbox_command, + _runtime_directories, run_supervised, ) +def test_runtime_directories_collapse_nested_but_keep_disjoint_roots( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stdlib = tmp_path / "python" / "lib" + purelib = stdlib / "site-packages" + platlib = tmp_path / "venv" / "site-packages" + purelib.mkdir(parents=True) + platlib.mkdir(parents=True) + monkeypatch.setattr( + "pdd.sync_core.supervisor.sysconfig.get_paths", + lambda: { + "stdlib": str(stdlib), + "platstdlib": str(stdlib), + "purelib": str(purelib), + "platlib": str(platlib), + }, + ) + + assert _runtime_directories() == ( + ("python-runtime/stdlib", stdlib), + ("python-runtime/platlib", platlib), + ) + + def test_linux_sandbox_uses_privileged_namespace_setup_then_drops_uid( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 4be27f9a03cb41c8cc71c6fd2e36cfcf4d29bade Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:08:42 -0700 Subject: [PATCH 119/128] fix(sync): restore protected imports and signer readiness --- pdd/sync_core/runner.py | 2 +- pdd/sync_core/signer_process.py | 15 ++++++++---- tests/test_sync_core_runner.py | 2 +- tests/test_sync_core_trust.py | 42 ++++++++++++++++++++++++--------- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 599cc333d..0c4675048 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -828,7 +828,7 @@ def _trusted_collection_runner( "", "os.chdir(_ROOT)", "_ENV = os.environ.copy()", - "_ENV['PYTHONPATH'] = _CONTROLLER", + "_ENV['PYTHONPATH'] = _CONTROLLER + os.pathsep + _ROOT", "_STATUS = subprocess.run([sys.executable, '-P', '-m', 'pytest'] + " + json.dumps(pytest_args) + ", env=_ENV).returncode", "sys.stdout.flush(); sys.stderr.flush()", diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 0e975c9f7..7cb390506 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -42,6 +42,9 @@ def cleanup(): time.sleep(.01) def stop(_signum, _frame): cleanup() + if child is not None: + stdout, stderr = child.communicate() + sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) raise SystemExit(124) signal.signal(signal.SIGTERM, stop) ready_path = os.environ.pop('PDD_SIGNER_READY_PATH', '') @@ -108,7 +111,9 @@ def _terminate_marked(token: str, leader: int, timeout: float = 0.5) -> None: pass -def _linux_contained_command(command: tuple[str, ...]) -> tuple[str, ...]: +def _linux_contained_command( + command: tuple[str, ...], writable_root: Path +) -> tuple[str, ...]: """Place the signer behind a PID namespace whose init owns all descendants.""" bwrap = shutil.which("bwrap") if bwrap is None: @@ -121,6 +126,7 @@ def _linux_contained_command(command: tuple[str, ...]) -> tuple[str, ...]: return (*prefix, bwrap, "--unshare-pid", "--die-with-parent", "--new-session", "--ro-bind", "/", "/", "--proc", "/proc", "--dev", "/dev", + "--bind", str(writable_root), str(writable_root), "--tmpfs", "/tmp", "--", sys.executable, "-c", _LINUX_CONTAINMENT, *command, ) @@ -158,13 +164,14 @@ def run_signer( ) -> subprocess.CompletedProcess[bytes]: """Run a signer in a new process group and reap the complete group on timeout.""" token = uuid.uuid4().hex - contained_command = ( - _linux_contained_command(command) if sys.platform.startswith("linux") else command - ) # The Linux signer scope replaces /tmp, so keep the checker-owned marker # under its home directory, which the signer PID namespace bind-mounts. with tempfile.TemporaryDirectory(prefix="pdd-signer-", dir=Path.home()) as directory: ready_path = Path(directory) / "started" + contained_command = ( + _linux_contained_command(command, Path(directory)) + if sys.platform.startswith("linux") else command + ) environment = os.environ | {"PDD_SIGNER_PROCESS_TOKEN": token} if sys.platform.startswith("linux"): environment["PDD_SIGNER_READY_PATH"] = str(ready_path) diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 2e9a0aadc..605ed863f 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -627,7 +627,7 @@ def test_collection_worker_uses_trusted_plugin_path(tmp_path: Path) -> None: assert "_CONTROLLER =" in source assert str(controller) in source - assert "_ENV['PYTHONPATH'] = _CONTROLLER" in source + assert "_ENV['PYTHONPATH'] = _CONTROLLER + os.pathsep + _ROOT" in source assert "[sys.executable, '-P', '-m', 'pytest']" in source diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index d297ff90e..ca4fb0b53 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -28,6 +28,7 @@ from pdd.sync_core.trust import ValidationEvidence from pdd.sync_core.finalize import attestation_signer_from_environment from pdd.sync_core.signer_process import run_signer +import pdd.sync_core.signer_process as signer_process_module from pdd.sync_core.certificate import RemoteCertificateSigner import pdd.sync_core.certificate as certificate_module import pdd.sync_core.trust as trust_module @@ -140,6 +141,24 @@ def test_signer_timeout_is_bounded_with_detached_pipe_holder() -> None: assert time.monotonic() - started < 1.5 +def test_linux_signer_containment_binds_only_ready_root_writable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + signer_process_module.shutil, + "which", + lambda name: "/usr/bin/bwrap" if name == "bwrap" else None, + ) + + command = signer_process_module._linux_contained_command(("signer",), tmp_path) + + assert command[command.index("--ro-bind") + 1:command.index("--ro-bind") + 3] == ( + "/", "/" + ) + bind_index = command.index("--bind") + assert command[bind_index + 1:bind_index + 3] == (str(tmp_path), str(tmp_path)) + + @pytest.mark.skipif( not sys.platform.startswith("linux"), reason="requires Linux stable process identity" ) @@ -147,26 +166,29 @@ def test_signer_timeout_is_bounded_with_detached_pipe_holder() -> None: def test_remote_signer_timeout_reaps_env_cleared_detached_descendant( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, adapter: str ) -> None: - pid_file = tmp_path / f"{adapter}.pid" marker = tmp_path / f"{adapter}.survived" detached = ( "import os,sys,time; " - "open(sys.argv[1], 'w').write(str(os.getpid())); " - "time.sleep(1); open(sys.argv[2], 'w').write('survived'); time.sleep(30)" + "time.sleep(1); open(sys.argv[1], 'w').write('survived'); time.sleep(30)" ) parent = ( "import os,subprocess,sys,time; env=dict(os.environ); " "env.pop('PDD_SIGNER_PROCESS_TOKEN', None); " - "subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2], sys.argv[3]], " + "child=subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " "env=env, start_new_session=True, stdout=subprocess.DEVNULL, " - "stderr=subprocess.DEVNULL); time.sleep(30)" + "stderr=subprocess.DEVNULL); print(child.pid, flush=True); time.sleep(30)" ) - command = (sys.executable, "-c", parent, detached, str(pid_file), str(marker)) + command = (sys.executable, "-c", parent, detached, str(marker)) actual_run_signer = run_signer + timed_out: list[subprocess.TimeoutExpired] = [] def short_run(command_value, payload, *, timeout): del timeout - return actual_run_signer(command_value, payload, timeout=0.1) + try: + return actual_run_signer(command_value, payload, timeout=0.1) + except subprocess.TimeoutExpired as exc: + timed_out.append(exc) + raise if adapter == "attestation": monkeypatch.setattr(trust_module, "run_signer", short_run) @@ -185,10 +207,8 @@ def short_run(command_value, payload, *, timeout): with pytest.raises(ValueError, match="timed out"): signer.sign_bytes(b"payload") - deadline = time.monotonic() + 1 - while not pid_file.exists() and time.monotonic() < deadline: - time.sleep(0.01) - pid = int(pid_file.read_text(encoding="utf-8")) + assert timed_out and timed_out[0].output + pid = int(timed_out[0].output.strip()) with pytest.raises(ProcessLookupError): os.kill(pid, 0) time.sleep(1.1) From 7baf2921b1d970ab6ff80f0ad529dcbe27737dd8 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:21:40 -0700 Subject: [PATCH 120/128] fix(sync): stream contained signer diagnostics --- pdd/sync_core/signer_process.py | 10 +++------- tests/test_sync_core_trust.py | 1 + 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 7cb390506..c7be139b9 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -42,19 +42,15 @@ def cleanup(): time.sleep(.01) def stop(_signum, _frame): cleanup() - if child is not None: - stdout, stderr = child.communicate() - sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) raise SystemExit(124) signal.signal(signal.SIGTERM, stop) ready_path = os.environ.pop('PDD_SIGNER_READY_PATH', '') -child = subprocess.Popen(sys.argv[1:], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, start_new_session=True) +child = subprocess.Popen(sys.argv[1:], stdin=subprocess.PIPE, stdout=sys.stdout.buffer, + stderr=sys.stderr.buffer, start_new_session=True) if ready_path: pathlib.Path(ready_path).touch(exist_ok=False) try: - stdout, stderr = child.communicate(sys.stdin.buffer.read()) - sys.stdout.buffer.write(stdout); sys.stderr.buffer.write(stderr) + child.communicate(sys.stdin.buffer.read()) status = child.returncode finally: cleanup() diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index ca4fb0b53..80faf06eb 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -157,6 +157,7 @@ def test_linux_signer_containment_binds_only_ready_root_writable( ) bind_index = command.index("--bind") assert command[bind_index + 1:bind_index + 3] == (str(tmp_path), str(tmp_path)) + assert "stdout=sys.stdout.buffer" in signer_process_module._LINUX_CONTAINMENT @pytest.mark.skipif( From 91e186c3bec93db5c132ac976af2eea7d227ba02 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:32:53 -0700 Subject: [PATCH 121/128] fix(sync): retain signer timeout diagnostics --- pdd/sync_core/signer_process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index c7be139b9..6dea4c42c 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -195,9 +195,10 @@ def run_signer( process.terminate() try: stdout, stderr = process.communicate(timeout=0.4) - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired as cleanup_exc: _kill_bounded(process, token) - stdout, stderr = b"", b"" + stdout = cleanup_exc.output or exc.output or b"" + stderr = cleanup_exc.stderr or exc.stderr or b"" else: _kill_bounded(process, token) stdout, stderr = b"", b"" From 2f5139a86bf869ae771564121f6cf7fada6348ae Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:43:16 -0700 Subject: [PATCH 122/128] test(sync): isolate protected signer containment smoke --- .github/workflows/unit-tests.yml | 6 ++++++ tests/test_sync_core_trust.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d9380d03e..10f1788db 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -117,6 +117,12 @@ jobs: tests/test_sync_core_runner.py::test_passing_collected_test_is_pass --timeout=60 + - name: Verify protected signer containment smoke + run: > + pytest -q + tests/test_sync_core_trust.py::test_remote_signer_timeout_reaps_env_cleared_detached_descendant + --timeout=60 + - name: Run focused protected-runner tests run: > pytest -q diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 80faf06eb..a024a1b81 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -186,7 +186,7 @@ def test_remote_signer_timeout_reaps_env_cleared_detached_descendant( def short_run(command_value, payload, *, timeout): del timeout try: - return actual_run_signer(command_value, payload, timeout=0.1) + return actual_run_signer(command_value, payload, timeout=0.5) except subprocess.TimeoutExpired as exc: timed_out.append(exc) raise From a2424de3f30d7d2ed00d87c298417ea3aa2dd8a3 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:48:29 -0700 Subject: [PATCH 123/128] test(sync): preserve signer startup failure diagnostics --- pdd/sync_core/signer_process.py | 7 ++++++- tests/test_sync_core_trust.py | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 6dea4c42c..6e3c3ac04 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -149,7 +149,12 @@ def _wait_for_signer_start( """Wait briefly for the containment init to launch the signer child.""" deadline = time.monotonic() + 0.5 while not ready_path.exists(): - if process.poll() is not None or time.monotonic() >= deadline: + if process.poll() is not None: + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired( + command, timeout, output=stdout, stderr=stderr + ) + if time.monotonic() >= deadline: _kill_bounded(process, token) raise subprocess.TimeoutExpired(command, timeout) time.sleep(0.005) diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index a024a1b81..0a3bb4352 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -208,7 +208,9 @@ def short_run(command_value, payload, *, timeout): with pytest.raises(ValueError, match="timed out"): signer.sign_bytes(b"payload") - assert timed_out and timed_out[0].output + assert timed_out and timed_out[0].output, ( + timed_out[0].stderr if timed_out else "signer did not time out" + ) pid = int(timed_out[0].output.strip()) with pytest.raises(ProcessLookupError): os.kill(pid, 0) From 3da5fddc8bace4eb15bd6405b03e76e69e4b0d5d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:53:10 -0700 Subject: [PATCH 124/128] fix(sync): expose signer readiness after tmp isolation --- pdd/sync_core/signer_process.py | 3 ++- tests/test_sync_core_trust.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/signer_process.py b/pdd/sync_core/signer_process.py index 6e3c3ac04..9bedd0ed0 100644 --- a/pdd/sync_core/signer_process.py +++ b/pdd/sync_core/signer_process.py @@ -122,8 +122,9 @@ def _linux_contained_command( return (*prefix, bwrap, "--unshare-pid", "--die-with-parent", "--new-session", "--ro-bind", "/", "/", "--proc", "/proc", "--dev", "/dev", + "--tmpfs", "/tmp", "--bind", str(writable_root), str(writable_root), - "--tmpfs", "/tmp", "--", sys.executable, "-c", _LINUX_CONTAINMENT, + "--", sys.executable, "-c", _LINUX_CONTAINMENT, *command, ) diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 0a3bb4352..3dbe44be5 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -157,6 +157,7 @@ def test_linux_signer_containment_binds_only_ready_root_writable( ) bind_index = command.index("--bind") assert command[bind_index + 1:bind_index + 3] == (str(tmp_path), str(tmp_path)) + assert command.index("--tmpfs") < bind_index assert "stdout=sys.stdout.buffer" in signer_process_module._LINUX_CONTAINMENT From 4ff7c612a2977a0ac770ffd8303e810a1c10a16d Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 13:57:57 -0700 Subject: [PATCH 125/128] test(sync): verify detached signer by host pid --- tests/test_sync_core_trust.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 3dbe44be5..99c31b175 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -178,7 +178,10 @@ def test_remote_signer_timeout_reaps_env_cleared_detached_descendant( "env.pop('PDD_SIGNER_PROCESS_TOKEN', None); " "child=subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " "env=env, start_new_session=True, stdout=subprocess.DEVNULL, " - "stderr=subprocess.DEVNULL); print(child.pid, flush=True); time.sleep(30)" + "stderr=subprocess.DEVNULL); " + "status=open(f'/proc/{child.pid}/status').read().splitlines(); " + "nspid=next(line for line in status if line.startswith('NSpid:')); " + "print(nspid.split()[1], flush=True); time.sleep(30)" ) command = (sys.executable, "-c", parent, detached, str(marker)) actual_run_signer = run_signer From 2d484247908d3e1ba5c93ac6f5287a84a81408e8 Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 14:02:50 -0700 Subject: [PATCH 126/128] test(sync): detect detached signer by command marker --- tests/test_sync_core_trust.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index 99c31b175..b42a8c6ec 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -178,10 +178,7 @@ def test_remote_signer_timeout_reaps_env_cleared_detached_descendant( "env.pop('PDD_SIGNER_PROCESS_TOKEN', None); " "child=subprocess.Popen([sys.executable, '-c', sys.argv[1], sys.argv[2]], " "env=env, start_new_session=True, stdout=subprocess.DEVNULL, " - "stderr=subprocess.DEVNULL); " - "status=open(f'/proc/{child.pid}/status').read().splitlines(); " - "nspid=next(line for line in status if line.startswith('NSpid:')); " - "print(nspid.split()[1], flush=True); time.sleep(30)" + "stderr=subprocess.DEVNULL); print('started', flush=True); time.sleep(30)" ) command = (sys.executable, "-c", parent, detached, str(marker)) actual_run_signer = run_signer @@ -215,9 +212,19 @@ def short_run(command_value, payload, *, timeout): assert timed_out and timed_out[0].output, ( timed_out[0].stderr if timed_out else "signer did not time out" ) - pid = int(timed_out[0].output.strip()) - with pytest.raises(ProcessLookupError): - os.kill(pid, 0) + assert timed_out[0].output.strip() == b"started" + marker_bytes = str(marker).encode() + escaped = [] + for process in Path("/proc").iterdir(): + if not process.name.isdigit(): + continue + try: + command_line = (process / "cmdline").read_bytes() + except (OSError, PermissionError): + continue + if marker_bytes in command_line: + escaped.append(process.name) + assert not escaped time.sleep(1.1) assert not marker.exists() From e54147b6a9c2cfb897dd334023c25b1638ca82bc Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 14:57:57 -0700 Subject: [PATCH 127/128] fix(sync): close pytest shadow and replay aliases --- pdd/sync_core/runner.py | 49 +++++++++------ pdd/sync_core/trust.py | 24 ++++--- tests/test_sync_core_runner.py | 110 ++++++++++++++++++++++++++++++++- tests/test_sync_core_trust.py | 30 ++++++++- 4 files changed, 181 insertions(+), 32 deletions(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index 0c4675048..bf45b0ed1 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -549,8 +549,10 @@ def runner_identity_digest( "checker_artifact_digest": _checker_artifact_digest(), "pytest_command": [ "", - "-m", - "pytest", + "-P", + "", + "import pytest", + "pytest.main", "-q", *PYTEST_PROTECTED_FLAGS, "", @@ -558,8 +560,10 @@ def runner_identity_digest( ], "pytest_collection_command": [ "", - "-m", - "pytest", + "-P", + "", + "import pytest", + "pytest.main", "--collect-only", "-q", "--strict-config", @@ -821,16 +825,16 @@ def _trusted_collection_runner( worker.write_text( "\n".join( ( - "import os, subprocess, sys", + "import os, sys", "", f"_ROOT = {json.dumps(str(root))}", f"_CONTROLLER = {json.dumps(str(directory))}", "", "os.chdir(_ROOT)", - "_ENV = os.environ.copy()", - "_ENV['PYTHONPATH'] = _CONTROLLER + os.pathsep + _ROOT", - "_STATUS = subprocess.run([sys.executable, '-P', '-m', 'pytest'] + " - + json.dumps(pytest_args) + ", env=_ENV).returncode", + "sys.path.insert(0, _CONTROLLER)", + "import pytest", + "sys.path.insert(0, _ROOT)", + "_STATUS = pytest.main(" + json.dumps(pytest_args) + ")", "sys.stdout.flush(); sys.stderr.flush()", "os._exit(_STATUS if _STATUS in (0, 5) else 125)", "", @@ -848,11 +852,13 @@ def _trusted_execution_runner( worker = directory / "execution_worker.py" worker.write_text( "\n".join(( - "import os, subprocess, sys", + "import os, sys", + f"_CONTROLLER = {json.dumps(str(directory))}", f"os.chdir({json.dumps(str(root))})", + "sys.path.insert(0, _CONTROLLER)", + "import pytest", f"sys.path.insert(0, {json.dumps(str(root))})", - "_STATUS = subprocess.run([sys.executable, '-m', 'pytest'] + " - + json.dumps(pytest_args) + ").returncode", + "_STATUS = pytest.main(" + json.dumps(pytest_args) + ")", "sys.stdout.flush(); sys.stderr.flush()", "os._exit(_STATUS if 0 <= _STATUS <= 5 else 125)", "", )), encoding="utf-8", @@ -876,14 +882,19 @@ def _run_test_node( node_id: str, timeout_seconds: int, ) -> RunnerExecution: - command = [ - sys.executable, - "-m", - "pytest", + pytest_args = [ "-q", *PYTEST_PROTECTED_FLAGS, node_id, ] + command = [ + sys.executable, + "-P", + "pdd-trusted-execution-runner", + "import-trusted-pytest", + "pytest.main", + *pytest_args, + ] command_digest = hashlib.sha256( json.dumps(command, separators=(",", ":")).encode() ).hexdigest() @@ -896,10 +907,10 @@ def _run_test_node( junit = controllers / f"result-{os.urandom(16).hex()}.xml" junit.touch(mode=0o600) worker = _trusted_execution_runner( - controllers, root, [*command[3:], f"--junitxml={junit}"] + controllers, root, [*pytest_args, f"--junitxml={junit}"] ) result, surviving = _managed_subprocess( - [sys.executable, str(worker)], cwd=controllers, + [sys.executable, "-P", str(worker)], cwd=controllers, timeout=timeout_seconds, env=_pytest_environment(home), writable_roots=(home.parent,), writable_files=(junit,), readable_roots=(root,), @@ -946,7 +957,7 @@ def _collect_node_ids( plugin_name, _plugin_directory = _trusted_probe_plugin(controllers) pytest_args.extend(("-p", plugin_name)) worker = _trusted_collection_runner(controllers, root, pytest_args) - command = [sys.executable, str(worker)] + command = [sys.executable, "-P", str(worker)] digest = hashlib.sha256( json.dumps( { diff --git a/pdd/sync_core/trust.py b/pdd/sync_core/trust.py index 86f3fc13b..0db4223ed 100644 --- a/pdd/sync_core/trust.py +++ b/pdd/sync_core/trust.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import hashlib import json import os import subprocess @@ -43,8 +44,8 @@ def issue(self, request: "AttestationRequest") -> "AttestationEnvelope": class ReplayStore: """Interface for atomic cross-verifier attestation nonce consumption.""" - def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: - """Bind a nonce to one attestation ID, allowing idempotent rechecks.""" + def consume(self, issuer: str, nonce: str, envelope_digest: str) -> None: + """Bind a nonce to one immutable signed envelope, allowing exact rechecks.""" raise NotImplementedError def is_durable(self) -> bool: @@ -58,15 +59,15 @@ class InMemoryReplayStore(ReplayStore): def __init__(self) -> None: self._seen: dict[tuple[str, str], str] = {} - def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: + def consume(self, issuer: str, nonce: str, envelope_digest: str) -> None: """Reject nonce reuse for a different signed statement.""" key = (issuer, nonce) previous = self._seen.get(key) if previous is not None: - if previous == attestation_id: + if previous == envelope_digest: return raise AttestationError("attestation nonce was replayed") - self._seen[key] = attestation_id + self._seen[key] = envelope_digest def is_durable(self) -> bool: """Return false for process-local test storage.""" @@ -120,7 +121,7 @@ def _write(self, payload: dict[str, str]) -> None: temporary.unlink(missing_ok=True) raise - def consume(self, issuer: str, nonce: str, attestation_id: str) -> None: + def consume(self, issuer: str, nonce: str, envelope_digest: str) -> None: """Atomically reject and record every repeated issuer/nonce pair.""" def consume_record(records): if not isinstance(records, dict) or not all( @@ -130,10 +131,10 @@ def consume_record(records): raise AttestationError("replay ledger has invalid records") key = base64.b64encode(f"{issuer}\0{nonce}".encode()).decode("ascii") previous = records.get(key) - if previous is not None and previous != attestation_id: + if previous is not None and previous != envelope_digest: raise AttestationError("attestation nonce was replayed") if previous is None: - records[key] = attestation_id + records[key] = envelope_digest return records error = None for _attempt in range(3): @@ -430,8 +431,13 @@ def _verify_freshness(self, envelope: AttestationEnvelope, now: datetime) -> Non raise AttestationError("attestation is expired") def _verify_replay(self, envelope: AttestationEnvelope) -> None: + envelope_digest = hashlib.sha256( + envelope.payload() + b"\0" + envelope.signature.encode("ascii") + ).hexdigest() self._replay_store.consume( - envelope.issuer, envelope.validity.nonce, envelope.attestation_id + envelope.issuer, + envelope.validity.nonce, + envelope_digest, ) def _verify( diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 605ed863f..6c35981ea 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -627,8 +627,114 @@ def test_collection_worker_uses_trusted_plugin_path(tmp_path: Path) -> None: assert "_CONTROLLER =" in source assert str(controller) in source - assert "_ENV['PYTHONPATH'] = _CONTROLLER + os.pathsep + _ROOT" in source - assert "[sys.executable, '-P', '-m', 'pytest']" in source + assert "PYTHONPATH" not in source + assert source.index("import pytest") < source.index("sys.path.insert(0, _ROOT)") + assert "_STATUS = pytest.main" in source + + +def test_worker_imports_trusted_pytest_before_candidate_root(tmp_path: Path) -> None: + root = tmp_path / "candidate" + controller = tmp_path / "controller" + root.mkdir() + controller.mkdir() + (root / "tests").mkdir() + (root / "product.py").write_text( + "def expected(): return 'candidate product'\n", encoding="utf-8" + ) + (root / "tests/test_widget.py").write_text( + "from pathlib import Path\n" + "import product\n" + "import pytest\n" + "def test_widget():\n" + " assert product.expected() == 'candidate product'\n" + " assert Path(pytest.__file__).resolve() != Path('pytest.py').resolve()\n", + encoding="utf-8", + ) + (root / "pytest.py").write_text( + "from pathlib import Path\n" + "Path('candidate-pytest-loaded').write_text('loaded')\n" + "def main(*_args, **_kwargs): return 0\n", + encoding="utf-8", + ) + worker = runner_module._trusted_execution_runner( + controller, root, ["-q", "tests/test_widget.py::test_widget"] + ) + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + + result = subprocess.run( + [sys.executable, "-P", str(worker)], cwd=controller, env=environment, + capture_output=True, text=True, check=False, + ) + + assert result.returncode == 0, result.stderr + assert not (root / "candidate-pytest-loaded").exists() + + +def test_candidate_pytest_module_cannot_forge_collection_or_junit_pass(tmp_path) -> None: + root = tmp_path / "candidate" + controller = tmp_path / "controller" + root.mkdir() + controller.mkdir() + (root / "tests").mkdir() + (root / "product.py").write_text("def expected(): return 1\n", encoding="utf-8") + (root / "tests/test_widget.py").write_text( + "import product\n\ndef test_widget(): assert product.expected() == 2\n", + encoding="utf-8", + ) + (root / "pytest.py").write_text( + "import sys\nfrom pathlib import Path\n" + "def forge(arguments):\n" + " Path('candidate-pytest-loaded').write_text('loaded')\n" + " if '--collect-only' in arguments:\n" + " print('PDD_PROTECTED_NODE_IDS=[\"forged::node\"]')\n" + " for argument in arguments:\n" + " if argument.startswith('--junitxml='):\n" + " Path(argument.split('=', 1)[1]).write_text(\n" + " ''\n" + " )\n" + " return 0\n" + "def main(arguments=None): return forge(arguments or [])\n" + "if __name__ == '__main__': raise SystemExit(forge(sys.argv[1:]))\n", + encoding="utf-8", + ) + plugin_name, _plugin_directory = runner_module._trusted_probe_plugin(controller) + collection = runner_module._trusted_collection_runner( + controller, + root, + [ + "--collect-only", "-q", "--strict-config", "--strict-markers", + "-p", "no:cacheprovider", "tests/test_widget.py", "-p", plugin_name, + ], + ) + junit = controller / "result.xml" + execution = runner_module._trusted_execution_runner( + controller, + root, + ["-q", "tests/test_widget.py::test_widget", f"--junitxml={junit}"], + ) + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + + collected = subprocess.run( + [sys.executable, "-P", str(collection)], cwd=controller, env=environment, + capture_output=True, text=True, check=False, + ) + executed = subprocess.run( + [sys.executable, "-P", str(execution)], cwd=controller, env=environment, + capture_output=True, text=True, check=False, + ) + + assert collected.returncode == 0, collected.stderr + assert "PDD_PROTECTED_NODE_IDS=[\"tests/test_widget.py::test_widget\"]" in ( + collected.stdout + ) + assert "forged::node" not in collected.stdout + assert executed.returncode == 1, executed.stderr + assert 'failures="1"' in junit.read_text(encoding="utf-8") + assert not (root / "candidate-pytest-loaded").exists() def test_execution_precreates_private_junit_channel( diff --git a/tests/test_sync_core_trust.py b/tests/test_sync_core_trust.py index b42a8c6ec..c92ac669a 100644 --- a/tests/test_sync_core_trust.py +++ b/tests/test_sync_core_trust.py @@ -420,6 +420,32 @@ def test_durable_nonce_collision_is_rejected_across_policy_instances(tmp_path) - assert path.stat().st_mode & 0o777 == 0o600 +def test_durable_same_attestation_id_with_distinct_signed_payload_is_rejected( + tmp_path, +) -> None: + path = tmp_path / "external/replay.json" + first = AttestationTrustPolicy( + {"trusted-ci": PUBLIC_KEY}, replay_store=FileReplayStore(path) + ) + second = AttestationTrustPolicy( + {"trusted-ci": PUBLIC_KEY}, replay_store=FileReplayStore(path) + ) + envelope = _envelope() + _verify(first, envelope) + conflicting = SIGNER.issue( + AttestationRequest( + envelope.attestation_id, + _binding(), + (ObligationEvidence("test", EvidenceOutcome.FAIL),), + envelope.validity.nonce, + NOW, + ) + ) + + with pytest.raises(AttestationError, match="replayed"): + _verify(second, conflicting) + + def test_durable_exact_statement_recheck_is_idempotent(tmp_path) -> None: path = tmp_path / "external/replay.json" envelope = _envelope() @@ -458,7 +484,7 @@ def test_file_replay_store_rejects_symlink_lock(tmp_path) -> None: path.with_name("replay.json.lock").symlink_to(outside) store = FileReplayStore(path) with pytest.raises(AttestationError, match="unsafe"): - store.consume("trusted-ci", "nonce", "attestation") + store.consume("trusted-ci", "nonce", "digest") def test_file_replay_store_rejects_symlinked_ancestor(tmp_path) -> None: @@ -469,4 +495,4 @@ def test_file_replay_store_rejects_symlinked_ancestor(tmp_path) -> None: (protected / "linked").symlink_to(outside, target_is_directory=True) store = FileReplayStore(protected / "linked" / "nested" / "replay.json") with pytest.raises(AttestationError, match="unsafe"): - store.consume("trusted-ci", "nonce", "attestation") + store.consume("trusted-ci", "nonce", "digest") From 48dd66202371c8f79beb6e565d3c378fb7ee162c Mon Sep 17 00:00:00 2001 From: Greg Tanaka Date: Sun, 12 Jul 2026 15:24:32 -0700 Subject: [PATCH 128/128] fix(sync): preload trusted collection probe --- pdd/sync_core/runner.py | 1 + tests/test_sync_core_runner.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pdd/sync_core/runner.py b/pdd/sync_core/runner.py index bf45b0ed1..f8a9b25dc 100644 --- a/pdd/sync_core/runner.py +++ b/pdd/sync_core/runner.py @@ -833,6 +833,7 @@ def _trusted_collection_runner( "os.chdir(_ROOT)", "sys.path.insert(0, _CONTROLLER)", "import pytest", + "import pdd_checker_pytest_probe", "sys.path.insert(0, _ROOT)", "_STATUS = pytest.main(" + json.dumps(pytest_args) + ")", "sys.stdout.flush(); sys.stderr.flush()", diff --git a/tests/test_sync_core_runner.py b/tests/test_sync_core_runner.py index 6c35981ea..5f124d4be 100644 --- a/tests/test_sync_core_runner.py +++ b/tests/test_sync_core_runner.py @@ -628,7 +628,9 @@ def test_collection_worker_uses_trusted_plugin_path(tmp_path: Path) -> None: assert "_CONTROLLER =" in source assert str(controller) in source assert "PYTHONPATH" not in source - assert source.index("import pytest") < source.index("sys.path.insert(0, _ROOT)") + assert source.index("import pytest") < source.index( + "import pdd_checker_pytest_probe" + ) < source.index("sys.path.insert(0, _ROOT)") assert "_STATUS = pytest.main" in source @@ -699,6 +701,12 @@ def test_candidate_pytest_module_cannot_forge_collection_or_junit_pass(tmp_path) "if __name__ == '__main__': raise SystemExit(forge(sys.argv[1:]))\n", encoding="utf-8", ) + (root / "pdd_checker_pytest_probe.py").write_text( + "from pathlib import Path\n" + "Path('candidate-fixed-probe-loaded').write_text('loaded')\n" + "def pytest_collection_modifyitems(items): items[:] = []\n", + encoding="utf-8", + ) plugin_name, _plugin_directory = runner_module._trusted_probe_plugin(controller) collection = runner_module._trusted_collection_runner( controller, @@ -735,6 +743,7 @@ def test_candidate_pytest_module_cannot_forge_collection_or_junit_pass(tmp_path) assert executed.returncode == 1, executed.stderr assert 'failures="1"' in junit.read_text(encoding="utf-8") assert not (root / "candidate-pytest-loaded").exists() + assert not (root / "candidate-fixed-probe-loaded").exists() def test_execution_precreates_private_junit_channel(