Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions docs/release-config-validation.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ python tools/validate_release_config.py \

Ready mode additionally checks:

- required metadata and gate fields do not contain placeholders;
- no placeholder string remains anywhere in the supplied record, including nested decision lists, evidence references, findings, and follow-through fields;
- `metadata.version` is a stable token rather than free-form prose;
- `metadata.evidence_cutoff` is a valid ISO date in `YYYY-MM-DD` form;
- the decision includes a non-empty rationale;
- at least one gate exists;
- passing and not-applicable gates cite evidence;
- not-applicable gates contain a scoped rationale;
Expand All @@ -44,6 +47,8 @@ Ready mode additionally checks:
- deferred decisions identify evidence gaps;
- do-not-release decisions identify a blocker or unresolved hard gate.

The version rule allows letters, digits, `.`, `_`, `+`, and `-`; it does not require semantic versioning. The purpose is to make the reviewed configuration identifiable and stable enough to compare with its evidence.

## Semantics

The configuration distinguishes:
Expand Down Expand Up @@ -71,7 +76,7 @@ The validator checks internal coherence. It does not determine whether:
python -m unittest discover -s tests -v
```

The tests cover blocker, condition, hard-gate, evidence, placeholder, deferred-decision, and duplicate-ID semantics.
The tests cover blocker, condition, hard-gate, evidence, nested-placeholder, date, version, deferred-decision, and duplicate-ID semantics.

## Customization

Expand Down
46 changes: 43 additions & 3 deletions tests/test_validate_release_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@
BASE = {
"metadata": {
"project": "Fictional Assistant",
"version": "0.1.0",
"version": "0.1.0-pilot",
"environment": "internal-pilot",
"decision_scope": "20 internal users; read-only tools",
"decision_owner": "Fictional Sponsor",
"evidence_cutoff": "2026-09-30",
},
"decision": {
"outcome": "release_with_conditions",
"rationale": "The fictional evidence supports only a bounded internal pilot.",
"blockers": [],
"required_actions": ["Complete the confirmatory sample before expansion."],
"conditions": ["Keep tools read-only."],
Expand Down Expand Up @@ -80,7 +81,7 @@ def test_pass_gate_requires_evidence(self) -> None:
def test_not_applicable_requires_rationale(self) -> None:
payload = copy.deepcopy(BASE)
payload["gates"][0]["status"] = "not_applicable"
payload["gates"][0]["limitation"] = ""
payload["gates"][0]["limitation"] = None
errors = validate(payload, "ready")
self.assertIn(
"gates[0] with status not_applicable must explain the scoped rationale in limitation",
Expand Down Expand Up @@ -112,12 +113,51 @@ def test_duplicate_gate_ids_are_rejected(self) -> None:
errors = validate(payload, "ready")
self.assertIn("duplicate gate id: AUTH-001", errors)

def test_ready_mode_rejects_placeholders(self) -> None:
def test_ready_mode_rejects_metadata_placeholder(self) -> None:
payload = copy.deepcopy(BASE)
payload["metadata"]["decision_owner"] = "[TBD]"
errors = validate(payload, "ready")
self.assertIn("metadata.decision_owner contains a placeholder", errors)

def test_ready_mode_rejects_nested_decision_placeholder(self) -> None:
payload = copy.deepcopy(BASE)
payload["decision"]["conditions"][0] = "[TBD: condition]"
errors = validate(payload, "ready")
self.assertIn("decision.conditions[0] contains a placeholder", errors)

def test_ready_mode_rejects_placeholder_evidence_reference(self) -> None:
payload = copy.deepcopy(BASE)
payload["gates"][0]["evidence"][0] = "REPLACE_WITH_EVIDENCE"
errors = validate(payload, "ready")
self.assertIn("gates[0].evidence[0] contains a placeholder", errors)

def test_ready_mode_requires_rationale(self) -> None:
payload = copy.deepcopy(BASE)
payload["decision"]["rationale"] = ""
errors = validate(payload, "ready")
self.assertIn("decision.rationale must be non-empty text in ready mode", errors)

def test_malformed_evidence_cutoff_is_rejected(self) -> None:
payload = copy.deepcopy(BASE)
payload["metadata"]["evidence_cutoff"] = "30 September 2026"
errors = validate(payload, "ready")
self.assertIn(
"metadata.evidence_cutoff must be a valid ISO date in YYYY-MM-DD form",
errors,
)

def test_unstable_version_identifier_is_rejected(self) -> None:
payload = copy.deepcopy(BASE)
payload["metadata"]["version"] = "pilot version 1"
errors = validate(payload, "ready")
self.assertTrue(any("metadata.version must be a stable identifier" in error for error in errors))

def test_template_mode_allows_placeholders(self) -> None:
payload = copy.deepcopy(BASE)
payload["metadata"]["decision_owner"] = "[TBD]"
payload["decision"]["conditions"][0] = "[TBD: condition]"
self.assertNotIn("metadata.decision_owner contains a placeholder", validate(payload, "template"))


if __name__ == "__main__":
unittest.main()
57 changes: 48 additions & 9 deletions tools/validate_release_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from __future__ import annotations

import argparse
from datetime import date
import re
import sys
from pathlib import Path
Expand Down Expand Up @@ -41,6 +42,7 @@
"not_applicable",
}
PLACEHOLDER = re.compile(r"(?:\[?TBD\]?|YOUR_|REPLACE_|<[^>]+>)", re.IGNORECASE)
VERSION_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,79}$")


def _load(path: str | Path) -> dict[str, Any]:
Expand All @@ -61,7 +63,33 @@ def _is_nonempty_text(value: Any) -> bool:


def _is_placeholder(value: Any) -> bool:
return not _is_nonempty_text(value) or bool(PLACEHOLDER.search(str(value)))
return _is_nonempty_text(value) and bool(PLACEHOLDER.search(value))


def _is_iso_date(value: Any) -> bool:
if not _is_nonempty_text(value):
return False
try:
date.fromisoformat(value)
except ValueError:
return False
return True


def _placeholder_paths(value: Any, path: str = "") -> list[str]:
"""Return paths to placeholder strings anywhere in a ready-mode payload."""
paths: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
child_path = f"{path}.{key}" if path else str(key)
paths.extend(_placeholder_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
child_path = f"{path}[{index}]"
paths.extend(_placeholder_paths(child, child_path))
elif _is_placeholder(value):
paths.append(path or "<root>")
return paths


def _require_list(mapping: dict[str, Any], key: str, errors: list[str]) -> list[Any]:
Expand Down Expand Up @@ -107,14 +135,10 @@ def _validate_gate(gate: Any, index: int, mode: str, errors: list[str]) -> None:
errors.append(f"{prefix}.evidence entries must be non-empty text")
if owner is not None and not _is_nonempty_text(owner):
errors.append(f"{prefix}.owner must be non-empty text")
if limitation is not None and not _is_nonempty_text(limitation):
errors.append(f"{prefix}.limitation must be non-empty text when provided")

if mode == "ready":
if _is_placeholder(gate_id):
errors.append(f"{prefix}.id contains a placeholder")
if _is_placeholder(question):
errors.append(f"{prefix}.question contains a placeholder")
if _is_placeholder(owner):
errors.append(f"{prefix}.owner contains a placeholder")
if status in {"pass", "not_applicable"} and not evidence:
errors.append(f"{prefix} with status {status} must cite evidence")
if status == "not_applicable" and not _is_nonempty_text(limitation):
Expand Down Expand Up @@ -152,8 +176,6 @@ def validate(payload: dict[str, Any], mode: str) -> list[str]:
for field in REQUIRED_METADATA:
if field in metadata and not _is_nonempty_text(metadata[field]):
errors.append(f"metadata.{field} must be non-empty text")
if mode == "ready" and field in metadata and _is_placeholder(metadata[field]):
errors.append(f"metadata.{field} contains a placeholder")

outcome = decision.get("outcome")
if outcome not in ALLOWED_OUTCOMES:
Expand Down Expand Up @@ -189,6 +211,23 @@ def validate(payload: dict[str, Any], mode: str) -> list[str]:
if mode == "template":
return errors

for path in _placeholder_paths(payload):
errors.append(f"{path} contains a placeholder")

version = metadata.get("version")
if _is_nonempty_text(version) and not VERSION_IDENTIFIER.fullmatch(version):
errors.append(
"metadata.version must be a stable identifier using letters, digits, '.', '_', '+', or '-'"
)

evidence_cutoff = metadata.get("evidence_cutoff")
if _is_nonempty_text(evidence_cutoff) and not _is_iso_date(evidence_cutoff):
errors.append("metadata.evidence_cutoff must be a valid ISO date in YYYY-MM-DD form")

rationale = decision.get("rationale")
if not _is_nonempty_text(rationale):
errors.append("decision.rationale must be non-empty text in ready mode")

if not gates:
errors.append("ready mode requires at least one gate")

Expand Down
Loading