Add optional variables and fix stdout/stderr ordering

.beads/issues.jsonl

@@ -1,5 +1,7 @@
 {"id":"cctr-0a7","title":"Add corpus setup/teardown tests","description":"Add tests for the corpus-level setup/teardown feature.\n\n## Location\n`test/corpus_setup/`\n\n## Test Cases\n\n1. **Corpus setup runs before suites**\n   - `_setup_corpus.txt` creates a marker file\n   - Suite setup verifies marker exists\n   \n2. **Corpus teardown runs after suites**\n   - Suite creates a marker\n   - `_teardown_corpus.txt` verifies suite ran and cleans up\n\n3. **Corpus setup failure skips all suites**\n   - `_setup_corpus.txt` with a failing test\n   - Verify suites are skipped\n\n4. **Corpus teardown runs even when suites fail**\n   - Suite with failing test\n   - `_teardown_corpus.txt` still executes\n\n5. **CCTR_CORPUS_WORK_DIR available to suites**\n   - Corpus setup writes to CCTR_CORPUS_WORK_DIR\n   - Suite reads from CCTR_CORPUS_WORK_DIR\n\n## Acceptance Criteria\n- All test cases pass\n- Tests serve as documentation for the feature","status":"closed","priority":1,"issue_type":"task","owner":"ajansson@cloudflare.com","created_at":"2026-01-26T00:26:34.690997+01:00","created_by":"Andreas Jansson","updated_at":"2026-01-26T00:42:37.913884+01:00","closed_at":"2026-01-26T00:42:37.913884+01:00","close_reason":"Scrapped - design issue with corpus root detection","dependencies":[{"issue_id":"cctr-0a7","depends_on_id":"cctr-k5x","type":"blocks","created_at":"2026-01-26T00:26:39.971625+01:00","created_by":"Andreas Jansson"},{"issue_id":"cctr-0a7","depends_on_id":"cctr-3hh","type":"blocks","created_at":"2026-01-26T00:26:40.08634+01:00","created_by":"Andreas Jansson"}]}
+{"id":"cctr-1rz","title":"Support # comment lines in corpus file headers","description":"Comment lines starting with # should be allowed at the top of corpus files (before and between file-level directives, and between the last directive and the first === test delimiter). Currently, a line like '# This tests the frobulator' between '%shell bash' and the first '===' causes the parser to stop and silently return 0 tests.\n\nThe fix is in cctr-corpus/src/lib.rs in the corpus_file() function. The skip_blank_lines calls (or a new skip_blank_and_comment_lines helper) should also consume lines starting with optional whitespace followed by '#'. The comment lines should be discarded.\n\nSpecifically, the while loop at line ~693 that checks '!peeked.starts_with(\"===\")' should also skip comment lines before giving up. And skip_blank_lines before and after directives should also skip comments.\n\nThis would allow corpus files to have documentation comments like:\n\n  %shell bash\n\n  # These tests verify the bridge protocol\n  # See docs/protocol.md for format details\n\n  ===\n  first test\n  ===","status":"open","priority":3,"issue_type":"task","owner":"ajansson@cloudflare.com","created_at":"2026-03-21T00:28:05.810823+01:00","created_by":"Andreas Jansson","updated_at":"2026-03-21T00:28:05.810823+01:00"}
 {"id":"cctr-2fp","title":"Improve constraint failure error messages","status":"closed","priority":1,"issue_type":"task","owner":"ajansson@cloudflare.com","created_at":"2026-01-23T21:31:21.141892+01:00","created_by":"Andreas Jansson","updated_at":"2026-01-23T21:45:26.383303+01:00","closed_at":"2026-01-23T21:45:26.383303+01:00","close_reason":"Implemented improved constraint failure error messages that show variable bindings"}
+{"id":"cctr-395","title":"Show warning when .txt file in test directory fails to parse","description":"When cctr discovers .txt files in a test suite directory, it silently ignores files that fail to parse. This makes it very hard to debug why tests aren't running. For example, if a file has comment lines (# ...) between the %shell directive and the first === test header, the parser stops and returns 0 tests. The file is silently treated as having no tests.\n\nThe fix: in list_tests() and run_suite(), when parse_file() returns Ok but with 0 tests, OR when parse_file() returns Err, emit a warning to stderr like:\n\n  warning: test/cli/basic.txt: 0 tests found (parse stopped at line 3: expected '===' delimiter)\n\nThe relevant code paths are:\n- discover.rs: corpus_files() returns the file paths\n- main.rs:182 list_tests() calls parse_file() per corpus file\n- runner.rs:732 run_suite() iterates corpus_files() and calls run_corpus_file()\n- cctr-corpus/src/lib.rs:657 corpus_file() parser - when the while loop breaks because input doesn't start with '===', the remaining unparsed input and line number are available in ParseState but discarded","status":"open","priority":2,"issue_type":"task","owner":"ajansson@cloudflare.com","created_at":"2026-03-21T00:27:49.555329+01:00","created_by":"Andreas Jansson","updated_at":"2026-03-21T00:27:49.555329+01:00"}
 {"id":"cctr-3hh","title":"Run corpus teardown after all suites complete","description":"Update the main runner to execute `_teardown_corpus.txt` once after all suites complete.\n\n## Location\n`crates/cctr/src/main.rs` (or runner orchestration)\n\n## Changes Needed\n\n1. After all suites complete (regardless of pass/fail), run corpus teardown\n2. Use the same corpus work directory from setup\n3. Report teardown results\n4. Clean up corpus temp directory after teardown\n\n## Error Handling\n- Corpus teardown runs even if suites failed\n- Teardown failures are reported but don't change overall exit code logic\n- (Or should teardown failure cause non-zero exit? TBD)\n\n## Acceptance Criteria\n- Corpus teardown runs after all suites\n- Teardown runs even when suites fail\n- Corpus work directory is cleaned up","status":"closed","priority":1,"issue_type":"task","owner":"ajansson@cloudflare.com","created_at":"2026-01-26T00:26:25.394876+01:00","created_by":"Andreas Jansson","updated_at":"2026-01-26T00:34:37.348448+01:00","closed_at":"2026-01-26T00:34:37.348448+01:00","close_reason":"Implemented","dependencies":[{"issue_id":"cctr-3hh","depends_on_id":"cctr-j9r","type":"blocks","created_at":"2026-01-26T00:26:39.85443+01:00","created_by":"Andreas Jansson"}]}
 {"id":"cctr-43k","title":"Support reading test from stdin with cctr -","status":"closed","priority":1,"issue_type":"feature","owner":"ajansson@cloudflare.com","created_at":"2026-01-23T17:18:30.307949+01:00","created_by":"Andreas Jansson","updated_at":"2026-01-23T17:21:53.963326+01:00","closed_at":"2026-01-23T17:21:53.963326+01:00","close_reason":"Closed"}
 {"id":"cctr-4oh","title":"Add JSON types: json_string, json_bool, json_array, json_object","status":"closed","priority":1,"issue_type":"feature","owner":"ajansson@cloudflare.com","created_at":"2026-01-23T16:00:49.052473+01:00","created_by":"Andreas Jansson","updated_at":"2026-01-23T16:12:49.545177+01:00","closed_at":"2026-01-23T16:12:49.545177+01:00","close_reason":"Closed"}

.gitignore

@@ -1,4 +1,5 @@
 *.greger
+.greger/
 target
 TODO.md
 .DS_Store

README.md

@@ -41,6 +41,7 @@ See the [test/](https://github.com/andreasjansson/cctr/tree/main/test) directory
   - [Exit-only tests](#exit-only-tests)
   - [Multiline output](#multiline-output)
 - [Variables](#variables)
+  - [Optional variables](#optional-variables)
 - [Constraints](#constraints)
   - [Comparison operators](#comparison-operators)
   - [Arithmetic operators](#arithmetic-operators)
@@ -524,6 +525,40 @@ Access patterns:
 
 JSON values may contain `null`, which can be tested with `== null` or `type(x) == null`.
 
+### Optional variables
+
+Use the `optional` modifier to mark a variable that may or may not appear in the output. An optional variable must occupy an entire line by itself.
+
+```
+===
+test with optional progress line
+===
+./my-command
+---
+{{ progress: optional string }}
+result: {{ value: number }}
+---
+where
+* value > 0
+```
+
+This test passes whether or not a progress line appears before the result. If the output is `result: 42`, the optional line is skipped. If the output is `Processing...\nresult: 42`, the variable `progress` captures `Processing...`.
+
+The `optional` modifier works with any type: `optional number`, `optional string`, `optional json object`, etc. With no type specified, `{{ x: optional }}` uses duck typing.
+
+Multiple consecutive optional lines are supported:
+
+```
+===
+command with optional headers
+===
+./verbose-command
+---
+{{ line1: optional string }}
+{{ line2: optional string }}
+actual output
+```
+
 ## Constraints
 
 Add a `where` section to validate captured variables with expressions:

crates/cctr-corpus/src/lib.rs

@@ -61,6 +61,7 @@ pub enum VarType {
 pub struct VariableDecl {
     pub name: String,
     pub var_type: Option<VarType>,
+    pub optional: bool,
 }
 
 /// Skip directive - unconditional or conditional (with shell command)
@@ -272,20 +273,35 @@ const RESERVED_KEYWORDS: &[&str] = &[
     "array",
     "object",
     "env",
+    "optional",
 ];
 
 fn is_reserved_keyword(name: &str) -> bool {
     RESERVED_KEYWORDS.contains(&name)
 }
 
-fn parse_placeholder(content: &str) -> Result<(String, Option<VarType>), String> {
+fn parse_placeholder(content: &str) -> Result<(String, Option<VarType>, bool), String> {
     let content = content.trim();
-    let (name, var_type) = if let Some(colon_pos) = content.find(':') {
+    let (name, var_type, optional) = if let Some(colon_pos) = content.find(':') {
         let name = content[..colon_pos].trim().to_string();
         let type_str = content[colon_pos + 1..].trim();
-        (name, parse_type_annotation(type_str))
+        let (optional, type_str) = if let Some(rest) = type_str
+            .strip_prefix("optional")
+            .filter(|r| r.is_empty() || r.starts_with(' '))
+        {
+            let rest = rest.trim();
+            (true, rest)
+        } else {
+            (false, type_str)
+        };
+        let var_type = if type_str.is_empty() {
+            None
+        } else {
+            parse_type_annotation(type_str)
+        };
+        (name, var_type, optional)
     } else {
-        (content.to_string(), None)
+        (content.to_string(), None, false)
     };
 
     if is_reserved_keyword(&name) {
@@ -295,7 +311,7 @@ fn parse_placeholder(content: &str) -> Result<(String, Option<VarType>), String>
         ));
     }
 
-    Ok((name, var_type))
+    Ok((name, var_type, optional))
 }
 
 fn extract_variables_from_expected(expected: &str) -> Result<Vec<VariableDecl>, String> {
@@ -306,9 +322,13 @@ fn extract_variables_from_expected(expected: &str) -> Result<Vec<VariableDecl>,
     while let Some(start) = remaining.find("{{") {
         if let Some(end) = remaining[start..].find("}}") {
             let content = &remaining[start + 2..start + end];
-            let (name, var_type) = parse_placeholder(content)?;
+            let (name, var_type, optional) = parse_placeholder(content)?;
             if !name.is_empty() && seen.insert(name.clone()) {
-                variables.push(VariableDecl { name, var_type });
+                variables.push(VariableDecl {
+                    name,
+                    var_type,
+                    optional,
+                });
             }
             remaining = &remaining[start + end + 2..];
         } else {
@@ -1592,4 +1612,85 @@ hello
         assert_eq!(file.tests.len(), 1);
         assert!(!file.tests[0].require);
     }
+
+    #[test]
+    fn test_optional_string_variable() {
+        let content = r#"===
+optional test
+===
+some_command
+---
+{{ header: optional string }}
+fixed line
+"#;
+        let file = parse_test(content);
+        assert_eq!(file.tests.len(), 1);
+        assert_eq!(file.tests[0].variables.len(), 1);
+        assert_eq!(file.tests[0].variables[0].name, "header");
+        assert_eq!(file.tests[0].variables[0].var_type, Some(VarType::String));
+        assert!(file.tests[0].variables[0].optional);
+    }
+
+    #[test]
+    fn test_optional_number_variable() {
+        let content = r#"===
+optional number
+===
+some_command
+---
+{{ n: optional number }}
+result: done
+"#;
+        let file = parse_test(content);
+        assert_eq!(file.tests[0].variables[0].name, "n");
+        assert_eq!(file.tests[0].variables[0].var_type, Some(VarType::Number));
+        assert!(file.tests[0].variables[0].optional);
+    }
+
+    #[test]
+    fn test_optional_duck_typed() {
+        let content = r#"===
+optional duck
+===
+some_command
+---
+{{ x: optional }}
+footer
+"#;
+        let file = parse_test(content);
+        assert_eq!(file.tests[0].variables[0].name, "x");
+        assert_eq!(file.tests[0].variables[0].var_type, None);
+        assert!(file.tests[0].variables[0].optional);
+    }
+
+    #[test]
+    fn test_non_optional_variable() {
+        let content = r#"===
+regular var
+===
+some_command
+---
+{{ x: number }}
+"#;
+        let file = parse_test(content);
+        assert!(!file.tests[0].variables[0].optional);
+    }
+
+    #[test]
+    fn test_optional_json_object() {
+        let content = r#"===
+optional json
+===
+some_command
+---
+{{ data: optional json object }}
+result
+"#;
+        let file = parse_test(content);
+        assert_eq!(
+            file.tests[0].variables[0].var_type,
+            Some(VarType::JsonObject)
+        );
+        assert!(file.tests[0].variables[0].optional);
+    }
 }