diff --git a/.claw.json b/.claw.json new file mode 100644 index 0000000..e0e4c18 --- /dev/null +++ b/.claw.json @@ -0,0 +1,5 @@ +{ + "permissions": { + "defaultMode": "dontAsk" + } +} diff --git a/.gitignore b/.gitignore index 0f0377a..25798d3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,10 @@ /testing/out.s /compiler/out.s /compiler/out.wh +/testing/.claw/sessions/884576d23d2173fe +/testing/.sandbox-home +/testing/.sandbox-tmp +/testing/__pycache__ +/testing/.claw/sessions/884576d23d2173fe +/testing/.claw/sessions/884576d23d2173fe +/testing/__pycache__ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..03a556d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,12 @@ +# CLAUDE.md + +This file provides guidance to Claw Code (clawcode.dev) when working with code in this repository. + +## Detected stack +- No specific language markers were detected yet; document the primary language and verification commands once the project structure settles. +- Frameworks: none detected from the supported starter markers. + +## Working agreement +- Prefer small, reviewable changes and keep generated bootstrap files aligned with actual repo workflows. +- Keep shared defaults in `.claw.json`; reserve `.claw/settings.local.json` for machine-local overrides. +- Do not overwrite existing `CLAUDE.md` content automatically; update it intentionally when repo workflows change. diff --git a/compiler/src/codegen/codegen.rs b/compiler/src/codegen/codegen.rs index 478795a..e31f36e 100644 --- a/compiler/src/codegen/codegen.rs +++ b/compiler/src/codegen/codegen.rs @@ -68,7 +68,7 @@ impl CodeGenerator { self.wh_write_line(format!("assume (bit_shift <= (31 as ui32));").as_str(), 0); self.wh_write_line(format!("assume (bit_shift >= (0 as ui32));").as_str(), 0); self.wh_write_line(format!("assume (stmt >= (0 as ui32));").as_str(), 0); - self.wh_write_line(format!("assume (stmt <= ({} as ui32));",self.stmt).as_str(), 0); + self.wh_write_line(format!("assume (stmt <= ({} as ui32));",self.stmt-1).as_str(), 0); self.wh_write_line(format!("flip_mask = ((1 as ui32) << bit_shift);").as_str(), 0); self.wh_write_line("\nres = main(stmt, flip_mask);", 0); diff --git a/compiler/src/main.rs b/compiler/src/main.rs index 0db0148..70a0a6b 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -25,6 +25,9 @@ struct Args { #[arg(long)] hard: bool, + + #[arg(long)] + normal: bool, } fn main() { @@ -54,7 +57,9 @@ fn main() { let mut parser: Parser = Parser::new(_tokens); let _ast: AST = parser.parse_program(); let mut codegen: CodeGenerator = CodeGenerator::new(output,wh_output); + let mut hard = false; codegen.generate(_ast, args.hard); + println!("Lexing and parsing completed successfully."); diff --git a/compiler/test.trv b/compiler/test.trv new file mode 100644 index 0000000..76a67d5 --- /dev/null +++ b/compiler/test.trv @@ -0,0 +1,40 @@ +func main() -> Integer { + let authenticated : Integer = 0; + if authenticated != 0 { + return 77; + } + let cardPin1 : Integer = 1; + let cardPin2 : Integer = 2; + let cardPin3 : Integer = 3; + let cardPin4 : Integer = 4; + let userPin1 : Integer = 0; + let userPin2 : Integer = 0; + let userPin3 : Integer = 0; + let userPin4 : Integer = 0; + let pinsEqual : Integer = 0; + if pinsEqual != 0 { + return 77; + } + + if userPin1 == cardPin1 then { + pinsEqual = pinsEqual + 1; + } + + if userPin2 == cardPin2 then { + pinsEqual = pinsEqual + 1; + } + + if userPin3 == cardPin3 then { + pinsEqual = pinsEqual + 1; + } + + if userPin4 == cardPin4 then { + pinsEqual = pinsEqual + 1; + } + + if pinsEqual == 4 then { + authenticated = 1; + } + + return authenticated; +} diff --git a/testing/test_results/compiled_test_files/pinny.hard.s b/test_results/compiled_test_files/pinny.hard.s similarity index 100% rename from testing/test_results/compiled_test_files/pinny.hard.s rename to test_results/compiled_test_files/pinny.hard.s diff --git a/testing/test_results/compiled_test_files/pinny.hard.wh b/test_results/compiled_test_files/pinny.hard.wh similarity index 99% rename from testing/test_results/compiled_test_files/pinny.hard.wh rename to test_results/compiled_test_files/pinny.hard.wh index f45bbec..a74a5ef 100644 --- a/testing/test_results/compiled_test_files/pinny.hard.wh +++ b/test_results/compiled_test_files/pinny.hard.wh @@ -173,7 +173,7 @@ bit_shift = (? as ui32); assume (bit_shift <= (31 as ui32)); assume (bit_shift >= (0 as ui32)); assume (stmt >= (0 as ui32)); -assume (stmt <= (15 as ui32)); +assume (stmt <= (14 as ui32)); flip_mask = ((1 as ui32) << bit_shift); res = main(stmt, flip_mask); diff --git a/testing/test_results/compiled_test_files/pinny.s b/test_results/compiled_test_files/pinny.s similarity index 100% rename from testing/test_results/compiled_test_files/pinny.s rename to test_results/compiled_test_files/pinny.s diff --git a/testing/test_results/compiled_test_files/pinny.wh b/test_results/compiled_test_files/pinny.wh similarity index 98% rename from testing/test_results/compiled_test_files/pinny.wh rename to test_results/compiled_test_files/pinny.wh index 55fd9c0..d2765e9 100644 --- a/testing/test_results/compiled_test_files/pinny.wh +++ b/test_results/compiled_test_files/pinny.wh @@ -91,7 +91,7 @@ bit_shift = (? as ui32); assume (bit_shift <= (31 as ui32)); assume (bit_shift >= (0 as ui32)); assume (stmt >= (0 as ui32)); -assume (stmt <= (15 as ui32)); +assume (stmt <= (14 as ui32)); flip_mask = ((1 as ui32) << bit_shift); res = main(stmt, flip_mask); diff --git a/test_results/edited_test_files/pinny.edited.hard.s b/test_results/edited_test_files/pinny.edited.hard.s new file mode 100644 index 0000000..dc62604 --- /dev/null +++ b/test_results/edited_test_files/pinny.edited.hard.s @@ -0,0 +1,316 @@ +.syntax unified +.thumb + +.section .text +.global _start +.type _start, %function + +_start: + mov r9, #0 + mov r10, #1 + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #1 + bne countermeasure + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #0 + cmp r1, r0 + mov r0, #0 + it ne + movne r0, #1 + cmp r0, #0 + beq else_0 + add r9, r9, #1 + cmp r9, #2 + bne countermeasure + mov r0, #77 + mov r7, #1 + svc #0 + b endif_0 +else_0: +endif_0: + sub sp, sp, #4 + mov r0, #1 + str r0, [sp] + add r9, r9, #1 + cmp r9, #3 + bne countermeasure + sub sp, sp, #4 + mov r0, #2 + str r0, [sp] + add r9, r9, #1 + cmp r9, #4 + bne countermeasure + sub sp, sp, #4 + mov r0, #3 + str r0, [sp] + add r9, r9, #1 + cmp r9, #5 + bne countermeasure + sub sp, sp, #4 + mov r0, #4 + str r0, [sp] + add r9, r9, #1 + cmp r9, #6 + bne countermeasure + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #7 + bne countermeasure + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #8 + bne countermeasure + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #9 + bne countermeasure + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #10 + bne countermeasure + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #11 + bne countermeasure + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #0 + cmp r1, r0 + mov r0, #0 + it ne + movne r0, #1 + cmp r0, #0 + beq else_1 + add r9, r9, #1 + cmp r9, #12 + bne countermeasure + mov r0, #77 + mov r7, #1 + svc #0 + b endif_1 +else_1: +endif_1: + ldr r0, [sp, #16] + mov r1, r0 + ldr r0, [sp, #32] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_2 + add r9, r9, #1 + cmp r9, #13 + bne countermeasure + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #14 + bne countermeasure + b endif_2 +else_2: +endif_2: + ldr r0, [sp, #12] + mov r1, r0 + ldr r0, [sp, #28] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_3 + add r9, r9, #1 + cmp r9, #15 + bne countermeasure + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #16 + bne countermeasure + b endif_3 +else_3: +endif_3: + ldr r0, [sp, #8] + mov r1, r0 + ldr r0, [sp, #24] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_4 + add r9, r9, #1 + cmp r9, #17 + bne countermeasure + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #18 + bne countermeasure + b endif_4 +else_4: +endif_4: + ldr r0, [sp, #4] + mov r1, r0 + ldr r0, [sp, #20] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_5 + add r9, r9, #1 + cmp r9, #19 + bne countermeasure + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + add r9, r9, #1 + cmp r9, #20 + bne countermeasure + b endif_5 +else_5: +endif_5: + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #4 + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_6 + add r9, r9, #1 + cmp r9, #21 + bne countermeasure + mov r0, #1 + str r0, [sp, #36] + add r9, r9, #1 + cmp r9, #22 + bne countermeasure + b endif_6 +else_6: +endif_6: + ldr r0, [sp, #36] + mov r7, #1 + svc #0 +countermeasure: + mov r0, #1 + ldr r1, =.Lstr0 + mov r2, #14 + mov r7, #4 + svc #0 + mov r0, #77 + mov r7, #1 + svc #0 + +.size _start, .-_start + +.section .data +.Lstr0: + .ascii "COUNTERMEASURE" +step_counter: + .word 0 +fault_msg: + .ascii "Control flow violation detected\n" +_metadata: + .word 0x10000000 @ 0 + .word 0xa+3 + .word 0xb+8 + .word 0x0 @ [r0] + .word 0x00000001 @ authenticated + .word 0x10000000 @ 1 + .word 0xa+27 + .word 0xb+32 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin1 + .word 0x10000000 @ 2 + .word 0xa+33 + .word 0xb+38 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin2 + .word 0x10000000 @ 3 + .word 0xa+39 + .word 0xb+44 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin3 + .word 0x10000000 @ 4 + .word 0xa+45 + .word 0xb+50 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin4 + .word 0x10000000 @ 5 + .word 0xa+51 + .word 0xb+56 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin1 + .word 0x10000000 @ 6 + .word 0xa+57 + .word 0xb+62 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin2 + .word 0x10000000 @ 7 + .word 0xa+63 + .word 0xb+68 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin3 + .word 0x10000000 @ 8 + .word 0xa+69 + .word 0xb+74 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin4 + .word 0x10000000 @ 9 + .word 0xa+75 + .word 0xb+80 + .word 0x0 @ [r0] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 10 + .word 0xa+111 + .word 0xb+118 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 11 + .word 0xa+134 + .word 0xb+141 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 12 + .word 0xa+157 + .word 0xb+164 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 13 + .word 0xa+180 + .word 0xb+187 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 14 + .word 0xa+203 + .word 0xb+207 + .word 0x0 @ [r0] + .word 0x00000001 @ authenticated diff --git a/test_results/edited_test_files/pinny.edited.hard.wh b/test_results/edited_test_files/pinny.edited.hard.wh new file mode 100644 index 0000000..b3b639f --- /dev/null +++ b/test_results/edited_test_files/pinny.edited.hard.wh @@ -0,0 +1,194 @@ +ui32 res; +ui32 stmt; +ui32 bit_shift; +ui32 flip_mask; + +fn main(ui32 stmt, ui32 flip_mask) -> ui32 { + ui32 authenticated; + ui32 cardPin1; + ui32 cardPin2; + ui32 cardPin3; + ui32 cardPin4; + ui32 userPin1; + ui32 userPin2; + ui32 userPin3; + ui32 userPin4; + ui32 pinsEqual; + ui32 step_counter; + step_counter = (0 as ui32); + authenticated = (0 as ui32); + if (stmt == (0 as ui32)) { + authenticated = (0 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (1 as ui32)) { + return (77 as ui32); + } + if ( authenticated != (0 as ui32)) { + step_counter++; + if (step_counter != (2 as ui32)) { + return (77 as ui32); + } + return ((77 as ui32) as ui32); + } + cardPin1 = (1 as ui32); + if (stmt == (1 as ui32)) { + cardPin1 = (1 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (3 as ui32)) { + return (77 as ui32); + } + cardPin2 = (2 as ui32); + if (stmt == (2 as ui32)) { + cardPin2 = (2 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (4 as ui32)) { + return (77 as ui32); + } + cardPin3 = (3 as ui32); + if (stmt == (3 as ui32)) { + cardPin3 = (3 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (5 as ui32)) { + return (77 as ui32); + } + cardPin4 = (4 as ui32); + if (stmt == (4 as ui32)) { + cardPin4 = (4 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (6 as ui32)) { + return (77 as ui32); + } + userPin1 = (0 as ui32); + if (stmt == (5 as ui32)) { + userPin1 = (0 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (7 as ui32)) { + return (77 as ui32); + } + userPin2 = (0 as ui32); + if (stmt == (6 as ui32)) { + userPin2 = (0 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (8 as ui32)) { + return (77 as ui32); + } + userPin3 = (0 as ui32); + if (stmt == (7 as ui32)) { + userPin3 = (0 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (9 as ui32)) { + return (77 as ui32); + } + userPin4 = (0 as ui32); + if (stmt == (8 as ui32)) { + userPin4 = (0 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (10 as ui32)) { + return (77 as ui32); + } + pinsEqual = (0 as ui32); + if (stmt == (9 as ui32)) { + pinsEqual = (0 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (11 as ui32)) { + return (77 as ui32); + } + if ( pinsEqual != (0 as ui32)) { + step_counter++; + if (step_counter != (12 as ui32)) { + return (77 as ui32); + } + return ((77 as ui32) as ui32); + } + if ( userPin1 == cardPin1) { + step_counter++; + if (step_counter != (13 as ui32)) { + return (77 as ui32); + } + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (10 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (14 as ui32)) { + return (77 as ui32); + } + } + if ( userPin2 == cardPin2) { + step_counter++; + if (step_counter != (15 as ui32)) { + return (77 as ui32); + } + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (11 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (16 as ui32)) { + return (77 as ui32); + } + } + if ( userPin3 == cardPin3) { + step_counter++; + if (step_counter != (17 as ui32)) { + return (77 as ui32); + } + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (12 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (18 as ui32)) { + return (77 as ui32); + } + } + if ( userPin4 == cardPin4) { + step_counter++; + if (step_counter != (19 as ui32)) { + return (77 as ui32); + } + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (13 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (20 as ui32)) { + return (77 as ui32); + } + } + if ( pinsEqual == (4 as ui32)) { + step_counter++; + if (step_counter != (21 as ui32)) { + return (77 as ui32); + } + authenticated = (1 as ui32); + if (stmt == (14 as ui32)) { + authenticated = (1 as ui32) ^ flip_mask; + } + step_counter++; + if (step_counter != (22 as ui32)) { + return (77 as ui32); + } + } + return (authenticated as ui32); +} +stmt = (? as ui32); +bit_shift = (? as ui32); +assume (bit_shift <= (31 as ui32)); +assume (bit_shift >= (0 as ui32)); +assume (stmt >= (0 as ui32)); +assume (stmt <= (14 as ui32)); +flip_mask = ((1 as ui32) << bit_shift); + +res = main(stmt, flip_mask); +assert(res == (0 as ui32)); diff --git a/test_results/edited_test_files/pinny.edited.s b/test_results/edited_test_files/pinny.edited.s new file mode 100644 index 0000000..0bec9db --- /dev/null +++ b/test_results/edited_test_files/pinny.edited.s @@ -0,0 +1,231 @@ +.syntax unified +.thumb + +.section .text +.global _start +.type _start, %function + +_start: + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #0 + cmp r1, r0 + mov r0, #0 + it ne + movne r0, #1 + cmp r0, #0 + beq else_0 + mov r0, #77 + mov r7, #1 + svc #0 + b endif_0 +else_0: +endif_0: + sub sp, sp, #4 + mov r0, #1 + str r0, [sp] + sub sp, sp, #4 + mov r0, #2 + str r0, [sp] + sub sp, sp, #4 + mov r0, #3 + str r0, [sp] + sub sp, sp, #4 + mov r0, #4 + str r0, [sp] + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + sub sp, sp, #4 + mov r0, #0 + str r0, [sp] + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #0 + cmp r1, r0 + mov r0, #0 + it ne + movne r0, #1 + cmp r0, #0 + beq else_1 + mov r0, #77 + mov r7, #1 + svc #0 + b endif_1 +else_1: +endif_1: + ldr r0, [sp, #16] + mov r1, r0 + ldr r0, [sp, #32] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_2 + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + b endif_2 +else_2: +endif_2: + ldr r0, [sp, #12] + mov r1, r0 + ldr r0, [sp, #28] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_3 + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + b endif_3 +else_3: +endif_3: + ldr r0, [sp, #8] + mov r1, r0 + ldr r0, [sp, #24] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_4 + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + b endif_4 +else_4: +endif_4: + ldr r0, [sp, #4] + mov r1, r0 + ldr r0, [sp, #20] + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_5 + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #1 + add r0, r1, r0 + str r0, [sp] + b endif_5 +else_5: +endif_5: + ldr r0, [sp, #0] + mov r1, r0 + mov r0, #4 + cmp r1, r0 + mov r0, #0 + it eq + moveq r0, #1 + cmp r0, #0 + beq else_6 + mov r0, #1 + str r0, [sp, #36] + b endif_6 +else_6: +endif_6: + ldr r0, [sp, #36] + mov r7, #1 + svc #0 + +.size _start, .-_start +_metadata: + .word 0x10000000 @ 0 + .word 0xa+1 + .word 0xb+3 + .word 0x0 @ [r0] + .word 0x00000001 @ authenticated + .word 0x10000000 @ 1 + .word 0xa+19 + .word 0xb+21 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin1 + .word 0x10000000 @ 2 + .word 0xa+22 + .word 0xb+24 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin2 + .word 0x10000000 @ 3 + .word 0xa+25 + .word 0xb+27 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin3 + .word 0x10000000 @ 4 + .word 0xa+28 + .word 0xb+30 + .word 0x0 @ [r0] + .word 0x00000001 @ cardPin4 + .word 0x10000000 @ 5 + .word 0xa+31 + .word 0xb+33 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin1 + .word 0x10000000 @ 6 + .word 0xa+34 + .word 0xb+36 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin2 + .word 0x10000000 @ 7 + .word 0xa+37 + .word 0xb+39 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin3 + .word 0x10000000 @ 8 + .word 0xa+40 + .word 0xb+42 + .word 0x0 @ [r0] + .word 0x00000001 @ userPin4 + .word 0x10000000 @ 9 + .word 0xa+43 + .word 0xb+45 + .word 0x0 @ [r0] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 10 + .word 0xa+70 + .word 0xb+74 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 11 + .word 0xa+87 + .word 0xb+91 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 12 + .word 0xa+104 + .word 0xb+108 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 13 + .word 0xa+121 + .word 0xb+125 + .word 0x0 @ [r0,r1] + .word 0x00000001 @ pinsEqual + .word 0x10000000 @ 14 + .word 0xa+138 + .word 0xb+139 + .word 0x0 @ [r0] + .word 0x00000001 @ authenticated diff --git a/test_results/edited_test_files/pinny.edited.wh b/test_results/edited_test_files/pinny.edited.wh new file mode 100644 index 0000000..361e57c --- /dev/null +++ b/test_results/edited_test_files/pinny.edited.wh @@ -0,0 +1,104 @@ +ui32 res; +ui32 stmt; +ui32 bit_shift; +ui32 flip_mask; + +fn main(ui32 stmt, ui32 flip_mask) -> ui32 { + ui32 authenticated; + ui32 cardPin1; + ui32 cardPin2; + ui32 cardPin3; + ui32 cardPin4; + ui32 userPin1; + ui32 userPin2; + ui32 userPin3; + ui32 userPin4; + ui32 pinsEqual; + authenticated = (0 as ui32); + if (stmt == (0 as ui32)) { + authenticated = (0 as ui32) ^ flip_mask; + } + if ( authenticated != (0 as ui32)) { + return ((77 as ui32) as ui32); + } + cardPin1 = (1 as ui32); + if (stmt == (1 as ui32)) { + cardPin1 = (1 as ui32) ^ flip_mask; + } + cardPin2 = (2 as ui32); + if (stmt == (2 as ui32)) { + cardPin2 = (2 as ui32) ^ flip_mask; + } + cardPin3 = (3 as ui32); + if (stmt == (3 as ui32)) { + cardPin3 = (3 as ui32) ^ flip_mask; + } + cardPin4 = (4 as ui32); + if (stmt == (4 as ui32)) { + cardPin4 = (4 as ui32) ^ flip_mask; + } + userPin1 = (0 as ui32); + if (stmt == (5 as ui32)) { + userPin1 = (0 as ui32) ^ flip_mask; + } + userPin2 = (0 as ui32); + if (stmt == (6 as ui32)) { + userPin2 = (0 as ui32) ^ flip_mask; + } + userPin3 = (0 as ui32); + if (stmt == (7 as ui32)) { + userPin3 = (0 as ui32) ^ flip_mask; + } + userPin4 = (0 as ui32); + if (stmt == (8 as ui32)) { + userPin4 = (0 as ui32) ^ flip_mask; + } + pinsEqual = (0 as ui32); + if (stmt == (9 as ui32)) { + pinsEqual = (0 as ui32) ^ flip_mask; + } + if ( pinsEqual != (0 as ui32)) { + return ((77 as ui32) as ui32); + } + if ( userPin1 == cardPin1) { + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (10 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + } + if ( userPin2 == cardPin2) { + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (11 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + } + if ( userPin3 == cardPin3) { + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (12 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + } + if ( userPin4 == cardPin4) { + pinsEqual = pinsEqual + (1 as ui32); + if (stmt == (13 as ui32)) { + pinsEqual = pinsEqual + (1 as ui32) ^ flip_mask; + } + } + if ( pinsEqual == (4 as ui32)) { + authenticated = (1 as ui32); + if (stmt == (14 as ui32)) { + authenticated = (1 as ui32) ^ flip_mask; + } + } + return (authenticated as ui32); +} +stmt = (? as ui32); +bit_shift = (? as ui32); +assume (bit_shift <= (31 as ui32)); +assume (bit_shift >= (0 as ui32)); +assume (stmt >= (0 as ui32)); +assume (stmt <= (14 as ui32)); +flip_mask = ((1 as ui32) << bit_shift); + +res = main(stmt, flip_mask); +assert(res == (0 as ui32)); diff --git a/testing/test_results/plots/exhaustive.svg b/test_results/plots/exhaustive.svg similarity index 62% rename from testing/test_results/plots/exhaustive.svg rename to test_results/plots/exhaustive.svg index 7e8fd99..61428e9 100644 --- a/testing/test_results/plots/exhaustive.svg +++ b/test_results/plots/exhaustive.svg @@ -6,7 +6,7 @@ - 2026-05-13T13:21:49.151067 + 2026-05-19T11:37:29.014955 image/svg+xml @@ -38,95 +38,111 @@ z " style="fill: #ffffff"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #e41a1c"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #e41a1c"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #377eb8"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #377eb8"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #4daf4a"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #4daf4a"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #984ea3"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #984ea3"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #ff7f00"/> - +" clip-path="url(#p49ed3ac4f7)" style="fill: #ff7f00"/> + + + + + + - - + @@ -239,7 +255,7 @@ z - + @@ -468,17 +484,17 @@ z - - + - + - + - + - + - + - + - + - + - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1182,27 +1226,223 @@ z - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1535,22 +1775,22 @@ z - - + - + - + @@ -1560,17 +1800,170 @@ z - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +" style="fill: #984ea3"/> - - - + + + + - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + +" style="fill: #ff7f00"/> - - - + + + - + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + +" style="fill: #ffff33"/> - - - + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - + diff --git a/test_results/plots/exhaustive_edited.svg b/test_results/plots/exhaustive_edited.svg new file mode 100644 index 0000000..7f319f8 --- /dev/null +++ b/test_results/plots/exhaustive_edited.svg @@ -0,0 +1,2142 @@ + + + + + + + + 2026-05-19T11:37:29.206517 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/exhaustive_fault_types_hard.svg b/test_results/plots/exhaustive_fault_types_hard.svg new file mode 100644 index 0000000..b217b55 --- /dev/null +++ b/test_results/plots/exhaustive_fault_types_hard.svg @@ -0,0 +1,2230 @@ + + + + + + + + 2026-05-19T11:37:29.837794 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/exhaustive_fault_types_normal.svg b/test_results/plots/exhaustive_fault_types_normal.svg new file mode 100644 index 0000000..0969ba3 --- /dev/null +++ b/test_results/plots/exhaustive_fault_types_normal.svg @@ -0,0 +1,2184 @@ + + + + + + + + 2026-05-19T11:37:29.636416 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/exhaustive_fault_types_normal_edited.svg b/test_results/plots/exhaustive_fault_types_normal_edited.svg new file mode 100644 index 0000000..48ee736 --- /dev/null +++ b/test_results/plots/exhaustive_fault_types_normal_edited.svg @@ -0,0 +1,2427 @@ + + + + + + + + 2026-05-19T11:37:30.042083 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/test_results/plots/guided.svg b/test_results/plots/guided.svg similarity index 73% rename from testing/test_results/plots/guided.svg rename to test_results/plots/guided.svg index e55561a..d997df9 100644 --- a/testing/test_results/plots/guided.svg +++ b/test_results/plots/guided.svg @@ -6,7 +6,7 @@ - 2026-05-13T13:21:49.294873 + 2026-05-19T11:37:29.351972 image/svg+xml @@ -38,95 +38,63 @@ z " style="fill: #ffffff"/> - +" clip-path="url(#p08e1b3b5a7)" style="fill: #e41a1c"/> - +" clip-path="url(#p08e1b3b5a7)" style="fill: #e41a1c"/> - +" clip-path="url(#p08e1b3b5a7)" style="fill: #377eb8"/> - +" clip-path="url(#p08e1b3b5a7)" style="fill: #377eb8"/> - +" clip-path="url(#p08e1b3b5a7)" style="fill: #4daf4a"/> - - - - - - - - - - - - - +" clip-path="url(#p08e1b3b5a7)" style="fill: #4daf4a"/> - - + @@ -239,7 +207,7 @@ z - + @@ -468,17 +436,17 @@ z - - + - - + + - - - + + + - + - - - - - - - - + + + + + + + + - - - - - - - - - - - - + - - - - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - + @@ -1020,27 +822,166 @@ z - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1354,199 +1295,247 @@ z - - + - - - + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + +" style="fill: #377eb8"/> - - - - - - - - + + + + - - - - - - - - - + - - - - - +" transform="scale(0.015625)"/> + + + + + + + - - + +" style="fill: #4daf4a"/> - - - + + + - - - - - - - - - - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + - + diff --git a/test_results/plots/guided_edited.svg b/test_results/plots/guided_edited.svg new file mode 100644 index 0000000..5a77ad4 --- /dev/null +++ b/test_results/plots/guided_edited.svg @@ -0,0 +1,1445 @@ + + + + + + + + 2026-05-19T11:37:29.480288 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/guided_fault_types_hard.svg b/test_results/plots/guided_fault_types_hard.svg new file mode 100644 index 0000000..471c8f1 --- /dev/null +++ b/test_results/plots/guided_fault_types_hard.svg @@ -0,0 +1,1593 @@ + + + + + + + + 2026-05-19T11:37:30.296612 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/guided_fault_types_normal.svg b/test_results/plots/guided_fault_types_normal.svg new file mode 100644 index 0000000..3e40e31 --- /dev/null +++ b/test_results/plots/guided_fault_types_normal.svg @@ -0,0 +1,1457 @@ + + + + + + + + 2026-05-19T11:37:30.182039 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/guided_fault_types_normal_edited.svg b/test_results/plots/guided_fault_types_normal_edited.svg new file mode 100644 index 0000000..c0a7c35 --- /dev/null +++ b/test_results/plots/guided_fault_types_normal_edited.svg @@ -0,0 +1,1618 @@ + + + + + + + + 2026-05-19T11:37:30.400464 + image/svg+xml + + + Matplotlib v3.10.9, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test_results/plots/symex_results.csv b/test_results/plots/symex_results.csv new file mode 100644 index 0000000..2852f76 --- /dev/null +++ b/test_results/plots/symex_results.csv @@ -0,0 +1,11 @@ +id,run_id,test,variant,stmt,faulty_bit,result,edited +1,1,pinny,Hard,9,2,77,0 +2,1,pinny,Hard,8,2,77,0 +3,1,pinny,Hard,6,1,77,0 +4,1,pinny,Hard,4,2,77,0 +5,1,pinny,Hard,2,1,77,0 +6,1,pinny,Hard,0,0,1,0 +7,1,pinny,Normal,9,2,1,0 +8,1,pinny,Normal,0,0,1,0 +9,1,pinny.edited,Normal,9,0,77,1 +10,1,pinny.edited,Normal,0,0,77,1 diff --git a/testing/test_results/results.db b/test_results/results.db similarity index 57% rename from testing/test_results/results.db rename to test_results/results.db index 2cae9e2..eae78e7 100644 Binary files a/testing/test_results/results.db and b/test_results/results.db differ diff --git a/test_results/tests/pinny.edited.trv b/test_results/tests/pinny.edited.trv new file mode 100644 index 0000000..76a67d5 --- /dev/null +++ b/test_results/tests/pinny.edited.trv @@ -0,0 +1,40 @@ +func main() -> Integer { + let authenticated : Integer = 0; + if authenticated != 0 { + return 77; + } + let cardPin1 : Integer = 1; + let cardPin2 : Integer = 2; + let cardPin3 : Integer = 3; + let cardPin4 : Integer = 4; + let userPin1 : Integer = 0; + let userPin2 : Integer = 0; + let userPin3 : Integer = 0; + let userPin4 : Integer = 0; + let pinsEqual : Integer = 0; + if pinsEqual != 0 { + return 77; + } + + if userPin1 == cardPin1 then { + pinsEqual = pinsEqual + 1; + } + + if userPin2 == cardPin2 then { + pinsEqual = pinsEqual + 1; + } + + if userPin3 == cardPin3 then { + pinsEqual = pinsEqual + 1; + } + + if userPin4 == cardPin4 then { + pinsEqual = pinsEqual + 1; + } + + if pinsEqual == 4 then { + authenticated = 1; + } + + return authenticated; +} diff --git a/testing/test_results/tests/pinny.trv b/test_results/tests/pinny.trv similarity index 100% rename from testing/test_results/tests/pinny.trv rename to test_results/tests/pinny.trv diff --git a/testing/test_results/tests/pinny.trv.output b/test_results/tests/pinny.trv.output similarity index 100% rename from testing/test_results/tests/pinny.trv.output rename to test_results/tests/pinny.trv.output diff --git a/testing/test_results/tests/pinny.wh b/test_results/tests/pinny.wh similarity index 100% rename from testing/test_results/tests/pinny.wh rename to test_results/tests/pinny.wh diff --git a/testing/.claw/sessions/884576d23d2173fe/session-1779103269871-0.jsonl b/testing/.claw/sessions/884576d23d2173fe/session-1779103269871-0.jsonl new file mode 100644 index 0000000..4a2e915 --- /dev/null +++ b/testing/.claw/sessions/884576d23d2173fe/session-1779103269871-0.jsonl @@ -0,0 +1,49 @@ +{"created_at_ms":1779103269870,"model":"nvidia/nemotron-3-super-120b-a12b:free","session_id":"session-1779103269871-0","type":"session_meta","updated_at_ms":1779103269870,"version":1,"workspace_root":"/home/pbk/school/P10-Master/testing"} +{"text":"look at @test, there is a comment saying \"# HERE: ....\" Please read that comment, and do what it says.","timestamp_ms":1779103290554,"type":"prompt_history"} +{"message":{"blocks":[{"text":"look at @test, there is a comment saying \"# HERE: ....\" Please read that comment, and do what it says.","type":"text"}],"role":"user"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-a81a84efa9ca0e8e","input":"{\"path\": \"test\"}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":7486,"output_tokens":39}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/test\",\n \"content\": \"#!/bin/python3\\n\\nimport argparse, os, toml, shutil, subprocess, sys\\nimport matplotlib.pyplot as plt\\nfrom datetime import datetime\\nfrom src.database import ResultsDatabase\\nfrom src.testrunner import TestRunner\\nfrom src.symexrunner import SymexRunner\\nfrom src.plotenator import Plotter\\nfrom rich.console import Console\\n\\n\\nclass Experiment:\\n def __init__(self, args, run_variants, config_path=None):\\n if config_path is None:\\n config_path = os.path.join(os.path.dirname(__file__), \\\"config.toml\\\")\\n\\n with open(config_path, \\\"r\\\") as f:\\n config = toml.load(f)\\n self.args = args\\n self.run_variants = run_variants\\n self.base_dir = os.path.dirname(__file__)\\n self.tests_folder = os.path.join(\\n self.base_dir, config.get(\\\"test_folder\\\", \\\"fissc\\\")\\n )\\n self.artifacts_folder = os.path.join(self.base_dir, \\\"artifacts\\\")\\n self.plot_mode = config.get(\\\"plotmode\\\", \\\"save\\\")\\n self.injection_points = config.get(\\n \\\"injection_points\\\",\\n [\\n \\\"r0\\\",\\n \\\"r1\\\",\\n \\\"r2\\\",\\n \\\"r3\\\",\\n \\\"r4\\\",\\n \\\"r5\\\",\\n \\\"r6\\\",\\n \\\"r7\\\",\\n \\\"r8\\\",\\n \\\"r9\\\",\\n \\\"r10\\\",\\n \\\"r11\\\",\\n \\\"r12\\\",\\n ],\\n )\\n self.default_pass_mode = config.get(\\\"pass_mode\\\", {}).get(\\n \\\"default\\\", \\\"returncode\\\"\\n )\\n self.pass_mode_configs = config.get(\\\"pass_mode\\\", {})\\n self.runner = TestRunner(config_path=args.config)\\n self.tests = self.runner._discover_tests()\\n self.compile_results = None\\n\\n def run(self):\\n\\n # check that the compiler and interpreter is present:\\n self.runner.check_bin()\\n\\n print(f\\\"Setting up...\\\")\\n folder = self.setup()\\n print(f\\\"Created folder: {folder}\\\")\\n\\n print(f\\\"\\\\nCompiling tests...\\\")\\n results = self.runner.compile()\\n print(f\\\"Compiled {len(results)} tests\\\")\\n all_fault_results = None\\n wh_files = sorted(\\n [f for f in os.listdir(self.runner.compiled_folder) if f.endswith(\\\".wh\\\")]\\n )\\n\\n print(\\\"Whiley Files are ready for adding fault injections...\\\")\\n print(\\\"\\\\nRunning symex...\\\")\\n symex_runner = SymexRunner(config_path=self.args.config)\\n symex_runner.artifacts_folder = self.runner.compiled_folder\\n symex_results = symex_runner.run(self.run_variants)\\n\\n self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\"\\\\nRunning Interpreter Guided...\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\"\\\\nRunning Interpreter Exhaustively...\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\"\\\\nSkipping test run (--skip-run)\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n\\n plotter = Plotter(self.db)\\n plots = plotter.create_plots()\\n\\n\\n plots_dir = os.path.join(self.run_folder, \\\"plots\\\")\\n os.makedirs(plots_dir, exist_ok=True)\\n\\n\\n def save_plot(name, fig):\\n \\\"\\\"\\\"Handles the saving of the figure to a specific directory.\\\"\\\"\\\"\\n path = os.path.join(plots_dir, f\\\"{name}.svg\\\")\\n fig.savefig(path, format=\\\"svg\\\", bbox_inches=\\\"tight\\\")\\n\\n # only these remain as plots\\n if self.plot_mode == \\\"save\\\" or self.plot_mode == \\\"both\\\":\\n save_plot(\\\"exhaustive\\\", plots[\\\"exhaustive\\\"])\\n save_plot(\\\"guided\\\", plots[\\\"guided\\\"])\\n\\n if self.plot_mode == \\\"show\\\" or self.plot_mode == \\\"both\\\":\\n plt.show()\\n for fig in plots.values():\\n plt.close(fig)\\n\\n # symex exported as CSV instead of plot\\n symex_csv_path = os.path.join(plots_dir, \\\"symex_results.csv\\\")\\n plotter.export_symex_csv(symex_csv_path)\\n\\n self.db.close()\\n \\n zip_base = os.path.join(self.artifacts_folder, self.run_name)\\n shutil.make_archive(\\n zip_base, \\\"zip\\\", self.artifacts_folder, self.run_folder_name\\n )\\n shutil.rmtree(self.run_folder)\\n\\n print(f\\\"\\\\nArtifacts saved to: {zip_base}.zip\\\")\\n\\n def setup(self):\\n timestamp = datetime.now().strftime(\\\"%Y%m%d_%H%M%S\\\")\\n run_num = self._get_next_run_number()\\n self.run_name = f\\\"{run_num}-{timestamp}\\\"\\n self.run_folder_name = f\\\"test_results\\\"\\n self.run_folder = os.path.join(self.artifacts_folder, self.run_folder_name)\\n self.run_tests_folder = os.path.join(self.run_folder, \\\"tests\\\")\\n self.runner.compiled_folder = os.path.join(\\n self.run_folder, \\\"compiled_test_files\\\"\\n )\\n db_path = os.path.join(self.run_folder, \\\"results.db\\\")\\n\\n os.makedirs(self.run_folder, exist_ok=True)\\n os.makedirs(self.run_tests_folder, exist_ok=True)\\n os.makedirs(self.runner.compiled_folder, exist_ok=True)\\n\\n for f in os.listdir(self.tests_folder):\\n if f.endswith(\\\".trv\\\") or f.endswith(\\\".trv.output\\\") or f.endswith(\\\".wh\\\"):\\n shutil.copy(os.path.join(self.tests_folder, f), self.run_tests_folder)\\n\\n self.db = ResultsDatabase(db_path)\\n self.run_id = self.db.create_run(run_num, self.tests_folder)\\n\\n return self.run_folder\\n\\n def _get_next_run_number(self):\\n if not os.path.exists(self.artifacts_folder):\\n return 1\\n\\n max_num = 0\\n for item in os.listdir(self.artifacts_folder):\\n base_name = item[:-4] if item.endswith(\\\".zip\\\") else item\\n if os.path.isdir(\\n os.path.join(self.artifacts_folder, item)\\n ) or item.endswith(\\\".zip\\\"):\\n parts = base_name.split(\\\"-\\\", 1)\\n if len(parts) == 2 and parts[0].isdigit():\\n max_num = max(max_num, int(parts[0]))\\n return max_num + 1\\n\\n\\ndef run_setup(target=\\\"all\\\", force=False):\\n script_dir = os.path.dirname(os.path.abspath(__file__))\\n setup_module = os.path.join(script_dir, \\\"src\\\", \\\"setup.py\\\")\\n\\n cmd = [sys.executable, setup_module, \\\"--target\\\", target]\\n if force:\\n cmd.append(\\\"-f\\\")\\n\\n result = subprocess.run(cmd)\\n return result.returncode\\n\\n\\ndef get_missing_binaries():\\n script_dir = os.path.dirname(os.path.abspath(__file__))\\n bin_dir = os.path.join(script_dir, \\\"bin\\\")\\n required_bins = [\\\"minimc\\\", \\\"compiler\\\", \\\"interpreter\\\"]\\n missing = []\\n for bin_name in required_bins:\\n if not os.path.isfile(os.path.join(bin_dir, bin_name)):\\n missing.append(bin_name)\\n return missing\\n\\n\\nif __name__ == \\\"__main__\\\":\\n parser = argparse.ArgumentParser()\\n parser.add_argument(\\\"--config\\\", type=str, default=None, help=\\\"Path to config.toml\\\")\\n parser.add_argument(\\n \\\"--setup\\\",\\n type=str,\\n nargs=\\\"?\\\",\\n const=\\\"all\\\",\\n default=None,\\n metavar=\\\"TARGETS\\\",\\n help=\\\"Only run setup (build dependencies), then quit. Comma-separated: compiler, interpreter, minimc, or all\\\",\\n )\\n parser.add_argument(\\n \\\"--skip-run\\\",\\n action=\\\"store_true\\\",\\n help=\\\"Only setup and compile, skip running tests\\\",\\n )\\n parser.add_argument(\\n \\\"--limit\\\",\\n type=int,\\n default=None,\\n help=\\\"Limit number of injection points to test\\\",\\n )\\n parser.add_argument(\\n \\\"--nohard\\\",\\n action=\\\"store_true\\\",\\n help=\\\"Skip hardened version when running tests\\\",\\n )\\n parser.add_argument(\\\"--hard\\\", action=\\\"store_true\\\", help=\\\"Only run hardened version\\\")\\n args, unknown = parser.parse_known_args()\\n\\n force = \\\"-f\\\" in unknown or \\\"--reinit\\\" in unknown\\n\\n if args.setup:\\n exit(run_setup(target=args.setup, force=force))\\n\\n missing_bins = get_missing_binaries()\\n if missing_bins:\\n print(f\\\"\\\\033[91mError: Binaries not found: {', '.join(missing_bins)}\\\\033[0m\\\")\\n if len(missing_bins) > 1:\\n print(\\\"Run 'test --setup' to compile the binaries\\\")\\n exit(1)\\n print(\\\"Run 'test --setup' to compile the binary\\\")\\n exit(1)\\n\\n\\n if args.nohard and args.hard:\\n print(\\\"Error: Cannot use both --nohard and --hard\\\")\\n exit()\\n\\n if args.hard:\\n run_variants = \\\"hard\\\"\\n elif args.nohard:\\n run_variants = \\\"normal\\\"\\n else:\\n run_variants = \\\"both\\\"\\n exp = Experiment(args, run_variants)\\n exp.run()\",\n \"numLines\": 253,\n \"startLine\": 1,\n \"totalLines\": 253\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-a81a84efa9ca0e8e","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-9c84382d6b5007ba","input":"{\"path\": \"src/symexrunner.py\"}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":10168,"output_tokens":396}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/src/symexrunner.py\",\n \"content\": \"import toml, os, subprocess, re\\n\\n\\nclass SymexRunner:\\n def __init__(self, config_path=None):\\n if config_path is None:\\n config_path = os.path.join(os.path.dirname(__file__), \\\"..\\\", \\\"config.toml\\\")\\n\\n with open(config_path, \\\"r\\\") as f:\\n config = toml.load(f)\\n\\n self.base_dir = os.path.join(os.path.dirname(__file__), \\\"..\\\")\\n self.tests_folder = os.path.join(\\n self.base_dir, config.get(\\\"test_folder\\\", \\\"fissc\\\")\\n )\\n self.artifacts_folder = os.path.join(self.base_dir, \\\"artifacts\\\")\\n bin_dir = os.path.join(os.path.dirname(__file__), \\\"..\\\", \\\"bin\\\")\\n self.minimc_path = os.path.join(bin_dir, \\\"minimc\\\")\\n def run(self, run_variants):\\n wh_files = sorted(\\n [\\n f\\n for f in os.listdir(self.artifacts_folder)\\n if f.endswith(\\\".wh\\\")\\n and (\\n run_variants == \\\"both\\\"\\n or (run_variants == \\\"normal\\\" and not f.endswith(\\\".hard.wh\\\"))\\n or (run_variants == \\\"hard\\\" and f.endswith(\\\".hard.wh\\\"))\\n )\\n ]\\n )\\n\\n all_results = []\\n\\n for wh_file in wh_files:\\n wh_path = os.path.join(self.artifacts_folder, wh_file)\\n print(f\\\"Running MiniMC on {wh_file}\\\")\\n cmd = [self.minimc_path, wh_path, \\\"mc\\\", \\\"--mc.symbolic\\\", \\\"--mc.all\\\", \\\"--mc.concretize\\\"]\\n\\n result = subprocess.run(\\n cmd,\\n cwd=self.tests_folder,\\n capture_output=True,\\n text=True,\\n env={**os.environ, \\\"TERM\\\": \\\"dumb\\\"}, # cleaner output\\n )\\n\\n violations = extract_violations(result.stdout)\\n parsed = [parse_violation(v) for v in violations]\\n variant = \\\"Normal\\\"\\n if wh_file.__contains__(\\\".hard.wh\\\"):\\n variant = \\\"Hard\\\"\\n\\n\\n for v in parsed:\\n all_results.append(\\n {\\n \\\"test\\\": wh_file.replace(\\\".hard.wh\\\", \\\"\\\").replace(\\\".wh\\\", \\\"\\\"),\\n \\\"variant\\\": variant,\\n \\\"stmt\\\": v.get(\\\"stmt\\\"),\\n \\\"faulty_bit\\\": v.get(\\\"bit_shift\\\"),\\n \\\"result\\\": v.get(\\\"res\\\"),\\n }\\n )\\n\\n return all_results\\n\\n\\ndef strip_ansi(text):\\n ansi_escape = re.compile(r\\\"\\\\x1B\\\\[[0-?]*[ -/]*[@-~]\\\")\\n return ansi_escape.sub(\\\"\\\", text)\\n\\n\\ndef extract_violations(stdout):\\n clean = strip_ansi(stdout)\\n\\n pattern = re.compile(r\\\"Found Violation\\\\s*\\\\n\\\\[.*?\\\\]\\\\s*\\\\n\\\\{.*?\\\\}\\\", re.DOTALL)\\n\\n return pattern.findall(clean)\\n\\n\\ndef parse_violation(block):\\n result = {}\\n\\n # extract key-value pairs like: prgm:res 128\\n matches = re.findall(r\\\"prgm:(\\\\w+)\\\\s+([^\\\\n]+)\\\", block)\\n\\n for key, value in matches:\\n # key: mem, value: <0 I8>\\n # key: res, value: <0x1 I32>\\n # key: stmt, value: <0x1 I32>\\n # key: flip_mask, value: <0x1 I32>\\n # key: bit_shift, value: <0 I32>\\n value = value.strip()\\n\\n inner = re.match(r\\\"<(\\\\S+)\\\\s+\\\\w+>\\\", value)\\n if inner:\\n raw = inner.group(1)\\n try:\\n value = int(raw, 0)\\n except ValueError:\\n pass\\n else:\\n try:\\n value = int(value)\\n except ValueError:\\n pass\\n\\n result[key] = value\\n\\n return result\\n\\n\\nif __name__ == \\\"__main__\\\":\\n runner = SymexRunner()\\n\\n print(f\\\"Running symex runner\\\")\\n runner.run()\",\n \"numLines\": 118,\n \"startLine\": 1,\n \"totalLines\": 118\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-9c84382d6b5007ba","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-aa06acdf5d445bcd","input":"{\"pattern\": \"fissc/*\"}","name":"glob_search","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":11358,"output_tokens":657}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"durationMs\": 0,\n \"numFiles\": 0,\n \"filenames\": [],\n \"truncated\": false\n}","tool_name":"glob_search","tool_use_id":"chatcmpl-tool-aa06acdf5d445bcd","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-8f335da783ca08f0","input":"{\"path\": \"config.toml\"}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":11433,"output_tokens":49}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/config.toml\",\n \"content\": \"test_folder = \\\"pinny\\\"\\nplotmode = \\\"both\\\" # show, save or both\\n\\ninjection_points = [\\n \\\"r0\\\", \\\"r1\\\", \\\"r2\\\", \\\"r3\\\", \\\"r4\\\", \\\"r5\\\", \\\"r6\\\", \\\"r7\\\",\\n \\\"r8\\\", \\\"r9\\\", \\\"r10\\\", \\\"r11\\\", \\\"r12\\\", \\\"pc=*\\\",\\n \\\"cpsr_n\\\"\\n]\\n\\n[pass_mode]\\ndefault = \\\"returncode\\\"\\n\\n[pass_mode.returncode]\\npassed = 0\\nfailed = 1\\npaniced = 2\\n\\n[pass_mode.stdout]\\nexpected_pass = \\\"g_authenticated: 0\\\"\\nexpected_fail = \\\"g_authenticated: 1\\\"\",\n \"numLines\": 20,\n \"startLine\": 1,\n \"totalLines\": 20\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-8f335da783ca08f0","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-84ce1d9b4f2513d8","input":"{\"pattern\": \"pinny/*\"}","name":"glob_search","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":11719,"output_tokens":70}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"durationMs\": 0,\n \"numFiles\": 5,\n \"filenames\": [\n \"/home/pbk/school/P10-Master/testing/pinny/pinny.wh\",\n \"/home/pbk/school/P10-Master/testing/pinny/disabled/crt-rsa.trv.output\",\n \"/home/pbk/school/P10-Master/testing/pinny/pinny.trv\",\n \"/home/pbk/school/P10-Master/testing/pinny/pinny.trv.output\",\n \"/home/pbk/school/P10-Master/testing/pinny/disabled/crt-rsa.trv\"\n ],\n \"truncated\": false\n}","tool_name":"glob_search","tool_use_id":"chatcmpl-tool-84ce1d9b4f2513d8","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-a91568ca95f5a0b3","input":"{\"path\": \"pinny/pinny.trv\"}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":11920,"output_tokens":122}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/pinny/pinny.trv\",\n \"content\": \"func main() -> Integer {\\n let authenticated : Integer = 0;\\n let cardPin1 : Integer = 1;\\n let cardPin2 : Integer = 2;\\n let cardPin3 : Integer = 3;\\n let cardPin4 : Integer = 4;\\n let userPin1 : Integer = 0;\\n let userPin2 : Integer = 0;\\n let userPin3 : Integer = 0;\\n let userPin4 : Integer = 0;\\n let pinsEqual : Integer = 0;\\n\\n if userPin1 == cardPin1 then {\\n pinsEqual = pinsEqual + 1;\\n }\\n\\n if userPin2 == cardPin2 then {\\n pinsEqual = pinsEqual + 1;\\n }\\n\\n if userPin3 == cardPin3 then {\\n pinsEqual = pinsEqual + 1;\\n }\\n\\n if userPin4 == cardPin4 then {\\n pinsEqual = pinsEqual + 1;\\n } \\n\\n if pinsEqual == 4 then {\\n authenticated = 1;\\n }\\n\\n return authenticated;\\n}\",\n \"numLines\": 34,\n \"startLine\": 1,\n \"totalLines\": 34\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-a91568ca95f5a0b3","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-899d30d165870174","input":"{\"path\": \"src/testrunner.py\"}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":12302,"output_tokens":651}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/src/testrunner.py\",\n \"content\": \"import os\\nimport re\\nimport subprocess\\nimport toml\\nimport concurrent.futures\\nimport multiprocessing\\nfrom tqdm import tqdm\\n\\ndef _get_countermeasure_location_range(asm_path: str):\\n with open(asm_path, \\\"r\\\") as f:\\n lines = f.readlines()\\n\\n start_pc = None\\n end_pc = None\\n current_pc = 0\\n found_start = False\\n \\n # We define what an 'instruction' looks like to ignore labels/comments\\n # Usually starts with whitespace followed by an opcode (mov, svc, etc.)\\n for line in lines:\\n clean_line = line.strip()\\n \\n # 1. Find the entry point\\n if \\\"_start:\\\" in clean_line:\\n found_start = True\\n current_pc += 1\\n continue\\n \\n if not found_start:\\n continue\\n\\n # 2. Check for the label we are looking for\\n if \\\"countermeasure:\\\" in clean_line:\\n start_pc = current_pc\\n\\n if start_pc is not None:\\n if \\\".size _start, .-_start\\\" in clean_line:\\n end_pc = current_pc -2\\n\\n\\n\\n current_pc += 1\\n\\n\\n\\n return start_pc, end_pc\\n\\n# Usage\\n# start, end = _get_countermeasure_location_range(\\\"code.s\\\")\\n# print(f\\\"Countermeasure range: {start} to {end}\\\")\\n\\ndef _fault_worker(args):\\n asm_path, injection_point, pass_mode_config = args\\n\\n bin_dir = os.path.join(os.path.dirname(__file__), \\\"..\\\", \\\"bin\\\")\\n interpreter_path = os.path.join(bin_dir, \\\"interpreter\\\")\\n\\n cmd = [interpreter_path, asm_path, \\\"--injection-point\\\", injection_point, \\\"--test-mode\\\"]\\n\\n result = subprocess.run(cmd, cwd=bin_dir, capture_output=True, text=True)\\n\\n (cm_start, cm_end) = _get_countermeasure_location_range(asm_path)\\n passed = _check_passed(result, pass_mode_config, cm_start, cm_end, injection_point)\\n\\n matched_output_line = None\\n if pass_mode_config.get(\\\"mode\\\") == \\\"stdout\\\":\\n expected_pass = pass_mode_config.get(\\\"expected_pass\\\", \\\"\\\")\\n expected_fail = pass_mode_config.get(\\\"expected_fail\\\", \\\"\\\")\\n \\n stdout_lines = result.stdout.strip().split(\\\"\\\\n\\\")\\n \\n if passed and expected_pass:\\n for line in stdout_lines:\\n if expected_pass in line:\\n matched_output_line = line.strip()\\n break\\n elif not passed and expected_fail:\\n for line in stdout_lines:\\n if expected_fail in line:\\n matched_output_line = line.strip()\\n break\\n \\n if matched_output_line is None:\\n matched_output_line = \\\"N/A\\\"\\n return {\\n \\\"injection_point\\\": injection_point,\\n \\\"returncode\\\": result.returncode,\\n \\\"stdout\\\": result.stdout.strip(),\\n \\\"stderr\\\": result.stderr.strip(),\\n \\\"expected_output\\\": pass_mode_config.get(\\\"expected_pass\\\", \\\"\\\"),\\n \\\"matched_output_line\\\": matched_output_line,\\n \\\"passed\\\": passed,\\n \\\"test_report\\\": _extract_test_report(result.stdout),\\n }\\n\\ndef _extract_test_report(stdout: str):\\n keyword = r'\\\"title\\\":\\\\s*\\\"test_report\\\"'\\n pattern = rf'\\\\{{[^{{}}]*{keyword}[^{{}}]*\\\\}}'\\n match = re.search(pattern, stdout,flags=re.MULTILINE)\\n if match: \\n return match.group(0)\\n return '{\\\"title\\\": \\\"test_report\\\"}'\\n \\ndef _check_passed(result, pass_mode_config, cm_start, cm_end, injection_point):\\n mode = pass_mode_config.get(\\\"mode\\\", \\\"returncode\\\")\\n expected_pass = pass_mode_config.get(\\\"expected_pass\\\", None)\\n expected_fail = pass_mode_config.get(\\\"expected_fail\\\", None)\\n\\n if \\\"COUNTERMEASURE\\\" in result.stdout:\\n return 77\\n elif \\\"Detected Infinite Loop\\\" in result.stdout:\\n return 3\\n if mode == \\\"returncode\\\" and result.returncode == 0:\\n return 1\\n elif mode == \\\"returncode\\\" and \\\"panic\\\" in result.stderr:\\n return 2\\n elif mode == \\\"returncode\\\" and result.returncode != 0:\\n if cm_start is not None and cm_end is not None and injection_point is not None:\\n parts = injection_point.split(\\\":\\\")\\n if parts[0] == \\\"pc\\\":\\n pc_value = int(parts[1])\\n bit_value = int(parts[2])\\n mask = 1 << bit_value\\n flipped_pc = pc_value ^ mask\\n if(flipped_pc in range(cm_start, cm_end+1)):\\n return 78\\n return 0\\n return 0\\n return 0\\n elif mode == \\\"stdout\\\":\\n if expected_pass and expected_pass in result.stdout:\\n return 1\\n elif expected_fail and expected_fail in result.stdout and result.returncode == 1:\\n return 0\\n return 1\\n else:\\n print(f\\\"STDOUT: {result.stdout}\\\")\\n print(f\\\"STDERR: {result.stderr}\\\")\\n print(f\\\"RETURNCODE: {result.returncode}\\\")\\n raise ValueError(f\\\"Unknown pass mode: {mode}\\\")\\n\\n\\nclass TestRunner:\\n def __init__(self, config_path=None):\\n if config_path is None:\\n config_path = os.path.join(os.path.dirname(__file__), \\\"..\\\", \\\"config.toml\\\")\\n\\n with open(config_path, \\\"r\\\") as f:\\n config = toml.load(f)\\n\\n self.base_dir = os.path.join(os.path.dirname(__file__), \\\"..\\\")\\n self.tests_folder = os.path.join(\\n self.base_dir, config.get(\\\"test_folder\\\", \\\"fissc\\\")\\n )\\n self.artifacts_folder = os.path.join(self.base_dir, \\\"artifacts\\\")\\n self.injection_points = config.get(\\n \\\"injection_points\\\",\\n [\\n \\\"r0\\\",\\n \\\"r1\\\",\\n \\\"r2\\\",\\n \\\"r3\\\",\\n \\\"r4\\\",\\n \\\"r5\\\",\\n \\\"r6\\\",\\n \\\"r7\\\",\\n \\\"r8\\\",\\n \\\"r9\\\",\\n \\\"r10\\\",\\n \\\"r11\\\",\\n \\\"r12\\\",\\n ],\\n )\\n self.default_pass_mode = config.get(\\\"pass_mode\\\", {}).get(\\\"default\\\", \\\"returncode\\\")\\n self.pass_mode_configs = config.get(\\\"pass_mode\\\", {})\\n self.tests = self._discover_tests()\\n self.compile_results = None\\n self.current_file : File = None\\n\\n def _discover_tests(self):\\n tests = []\\n if not os.path.exists(self.tests_folder):\\n return tests\\n\\n files = os.listdir(self.tests_folder)\\n trv_files = sorted([f for f in files if f.endswith(\\\".trv\\\")])\\n\\n for trv_file in trv_files:\\n test_name = trv_file[:-4]\\n output_file = trv_file + \\\".output\\\"\\n toml_file = trv_file + \\\".toml\\\"\\n\\n if output_file not in files:\\n continue\\n\\n trv_path = os.path.join(self.tests_folder, trv_file)\\n output_path = os.path.join(self.tests_folder, output_file)\\n\\n with open(output_path, \\\"r\\\") as f:\\n expected_output = f.read().strip()\\n\\n pass_mode_config = self._load_pass_mode_config(test_name, trv_file, files)\\n\\n tests.append(\\n {\\n \\\"name\\\": test_name,\\n \\\"path\\\": trv_path,\\n \\\"expected_output\\\": expected_output,\\n \\\"pass_mode\\\": pass_mode_config,\\n }\\n )\\n\\n return tests\\n\\n def _load_pass_mode_config(self, test_name, trv_file, files):\\n toml_file = trv_file + \\\".toml\\\"\\n\\n if toml_file in files:\\n toml_path = os.path.join(self.tests_folder, toml_file)\\n with open(toml_path, \\\"r\\\") as f:\\n test_config = toml.load(f)\\n pass_mode = test_config.get(\\\"pass_mode\\\", {})\\n if \\\"mode\\\" in pass_mode:\\n mode = pass_mode[\\\"mode\\\"]\\n mode_config = self.pass_mode_configs.get(mode, {})\\n return {\\n \\\"mode\\\": mode,\\n \\\"expected_pass\\\": pass_mode.get(\\\"passed\\\", 0) if mode == \\\"returncode\\\" else pass_mode.get(\\\"expected_pass\\\"),\\n \\\"expected_fail\\\": pass_mode.get(\\\"failed\\\", 1) if mode == \\\"returncode\\\" else pass_mode.get(\\\"expected_fail\\\"),\\n }\\n\\n mode = self.default_pass_mode\\n mode_config = self.pass_mode_configs.get(mode, {})\\n return {\\n \\\"mode\\\": mode,\\n \\\"expected_pass\\\": mode_config.get(\\\"passed\\\", 0) if mode == \\\"returncode\\\" else mode_config.get(\\\"expected_pass\\\"),\\n \\\"expected_fail\\\": mode_config.get(\\\"failed\\\", 1) if mode == \\\"returncode\\\" else mode_config.get(\\\"expected_fail\\\"),\\n }\\n\\n def check_bin(self):\\n from pathlib import Path\\n\\n script_dir = Path(__file__).resolve().parent.parent\\n setup_path = script_dir / \\\"setup\\\"\\n\\n bin_dir = script_dir / \\\"bin\\\"\\n\\n if not bin_dir.is_dir():\\n raise FileNotFoundError(\\n f\\\"Missing: compiler, interpreter. Fix by running: {setup_path}\\\"\\n )\\n\\n missing = []\\n\\n if not (bin_dir / \\\"compiler\\\").is_file():\\n missing.append(\\\"compiler\\\")\\n\\n if not (bin_dir / \\\"interpreter\\\").is_file():\\n missing.append(\\\"interpreter\\\")\\n\\n if not (bin_dir / \\\"minimc\\\").is_file():\\n missing.append(\\\"minimc\\\")\\n\\n if missing:\\n raise FileNotFoundError(\\n f\\\"Missing: {', '.join(missing)}. Fix by running: {setup_path}\\\"\\n )\\n\\n def compile_test(self, test, hard, compiled_folder):\\n trv_path = test[\\\"path\\\"]\\n test_name = test[\\\"name\\\"]\\n\\n if hard:\\n asm_path = os.path.join(compiled_folder, f\\\"{test_name}.hard\\\")\\n else:\\n asm_path = os.path.join(compiled_folder, f\\\"{test_name}\\\")\\n\\n compiler_dir = os.path.join(os.path.dirname(__file__), \\\"..\\\", \\\"bin\\\")\\n\\n cmd = [os.path.join(compiler_dir, \\\"compiler\\\"), trv_path, \\\"-o\\\", asm_path]\\n if hard:\\n cmd.append(\\\"--hard\\\")\\n\\n result = subprocess.run(cmd, cwd=compiler_dir, capture_output=True, text=True)\\n asm_path = asm_path + \\\".s\\\"\\n return result.returncode == 0, asm_path, result.stdout, result.stderr\\n\\n def interpret_test(self, asm_path, injection_point=None, debug=False):\\n bin_dir = os.path.join(os.path.dirname(__file__), \\\"..\\\", \\\"bin\\\")\\n interpreter_path = os.path.join(bin_dir, \\\"interpreter\\\")\\n\\n cmd = [interpreter_path, asm_path]\\n\\n if debug:\\n cmd.append(\\\"--debug\\\")\\n\\n if injection_point:\\n cmd.extend([\\\"--injection-point\\\", injection_point])\\n\\n result = subprocess.run(cmd, cwd=bin_dir, capture_output=True, text=True)\\n\\n return {\\n \\\"returncode\\\": result.returncode,\\n \\\"stdout\\\": result.stdout.strip(),\\n \\\"stderr\\\": result.stderr.strip(),\\n }\\n\\n def list_tests(self):\\n return self.tests\\n\\n\\n def _get_program_max_pc(self, asm_path):\\n with open(asm_path, \\\"r\\\") as f:\\n lines = f.readlines()\\n\\n in_start = False\\n instruction_count = 0\\n\\n for raw_line in lines:\\n line = raw_line.strip()\\n\\n if not line:\\n continue\\n\\n # remove comments\\n if \\\";\\\" in line:\\n line = line.split(\\\";\\\", 1)[0].strip()\\n if \\\"//\\\" in line:\\n line = line.split(\\\"//\\\", 1)[0].strip()\\n\\n if not line:\\n continue\\n\\n if not in_start:\\n if line == \\\"_start:\\\":\\n in_start = True\\n continue\\n\\n if line.startswith(\\\".\\\"):\\n continue\\n\\n # count everything else as an instruction\\n instruction_count += 1\\n\\n return instruction_count\\n\\n def _generate_injection_points(self, asm_path):\\n max_pc = self._get_program_max_pc(asm_path)\\n injection_points = []\\n\\n # reg faults: reg:::\\n registers = []\\n cpsr_registers = []\\n pc_points = []\\n\\n for point in self.injection_points:\\n if point.startswith('cpsr_'):\\n cpsr_registers.append(point[5:])\\n elif point.startswith('r') and point[1:].isdigit():\\n registers.append(point[1:])\\n elif point.startswith('pc=') and point[3] == \\\"*\\\":\\n pc_points.clear();\\n pc_points.append(\\\"ALL\\\")\\n elif point.startswith('pc=') and point[3:].isdigit():\\n if len(pc_points) > 0 and pc_points[0] == \\\"ALL\\\":\\n continue\\n pc_points.append(point[3:])\\n\\n\\n if len(pc_points) > 0:\\n if pc_points[0] == \\\"ALL\\\":\\n for pc in range(1, max_pc):\\n for bit in range(0, 32):\\n injection_points.append(f\\\"pc:{pc}:{bit}\\\")\\n else:\\n for pc in pc_points:\\n for bit in range(0, 32):\\n injection_points.append(f\\\"pc:{pc}:{bit}\\\")\\n\\n if len(registers) > 0:\\n for pc in range(1, max_pc):\\n for reg in registers:\\n for bit in range(0, 32):\\n injection_points.append(f\\\"reg:{pc}:{reg}:{bit}\\\")\\n\\n if len(cpsr_registers) > 0:\\n for pc in range(1, max_pc):\\n for reg in cpsr_registers:\\n injection_points.append(f\\\"cpsr:{pc}:{reg}\\\")\\n\\n return injection_points\\n\\n def stmt_to_pc_range(self, asm_path, stmt: int):\\n with open(asm_path, 'r') as f:\\n lines = f.readlines()\\n for i, line in enumerate(lines):\\n if f'.word 0x10000000 @ {stmt}' in line:\\n a_val = int(lines[i + 1].split('+')[1].split()[0])\\n b_val = int(lines[i + 2].split('+')[1].split()[0])\\n return a_val, b_val\\n return None\\n\\n def stmt_to_regs(self, asm_path, stmt: int):\\n with open(asm_path, 'r') as f:\\n lines = f.readlines()\\n for i, line in enumerate(lines):\\n if f'.word 0x10000000 @ {stmt}' in line:\\n return re.findall(r'r(\\\\d+)', lines[i + 3])\\n return None\\n\\n\\n def _symex_to_injection_points(self, variant, asm_path, minimc_res):\\n injection_points = []\\n for fault in minimc_res:\\n if fault['variant'].lower() != variant:\\n continue\\n stmt = fault['stmt']\\n (start_pc, end_pc) = self.stmt_to_pc_range(asm_path, stmt)\\n registers = self.stmt_to_regs(asm_path, stmt)\\n bit = fault['faulty_bit']\\n for reg in registers:\\n for pc in range(start_pc, end_pc+1):\\n ip = f\\\"reg:{pc}:{reg}:{bit}\\\"\\n injection_points.append(ip)\\n for pc in range(start_pc, end_pc+1):\\n for bit in range(0, 32):\\n ip = f\\\"pc:{pc}:{bit}\\\"\\n injection_points.append(ip)\\n for cpsr in [\\\"n\\\",\\\"v\\\",\\\"z\\\",\\\"c\\\"]:\\n ip = f\\\"cpsr:{pc}:{cpsr}\\\"\\n return injection_points\\n\\n\\n def _run_fault_campaign(self, test_name, variant, asm_path, pass_mode_config, limit=None, minimc_res=None):\\n exhaustive = True\\n if minimc_res:\\n exhaustive = False\\n\\n injection_points : list[str] = []\\n if exhaustive:\\n injection_points = self._generate_injection_points(asm_path)\\n else:\\n injection_points = self._symex_to_injection_points(variant, asm_path, minimc_res)\\n\\n if limit is not None and len(injection_points) > limit:\\n injection_points = injection_points[:limit]\\n print(f\\\" Limited to {limit} injection points\\\")\\n\\n cpu_count = multiprocessing.cpu_count()\\n print(f\\\" Using {cpu_count} workers\\\")\\n\\n results = []\\n\\n tasks = [\\n (asm_path, injection_point, pass_mode_config)\\n for injection_point in injection_points\\n ]\\n\\n with concurrent.f… [truncated for session JSONL]","tool_name":"read_file","tool_use_id":"chatcmpl-tool-899d30d165870174","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-a4a7992623b11909","input":"{\"path\": \"test\", \"old_string\": \" self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n\\n plotter = Plotter(self.db)\", \"new_string\": \" self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n print(f\\\"\\\\n{'='*60}\\\")\\n print(\\\"ERRORS FOUND BY SYMEX (Symbolic Execution)\\\")\\n print(f\\\"{'='*60}\\\")\\n if symex_results:\\n # Group results by test for better display\\n results_by_test = {}\\n for result in symex_results:\\n test_name = result[\\\"test\\\"]\\n if test_name not in results_by_test:\\n results_by_test[test_name] = []\\n results_by_test[test_name].append(result)\\n \\n for test_name, results in results_by_test.items():\\n print(f\\\"\\\\nTest: {test_name}\\\")\\n print(\\\"-\\\" * 40)\\n # Find the .trv file for this test\\n trv_path = None\\n for test in self.tests:\\n if test[\\\"name\\\"] == test_name:\\n trv_path = test[\\\"path\\\"]\\n break\\n \\n if trv_path and os.path.exists(trv_path):\\n try:\\n with open(trv_path, 'r') as f:\\n trv_lines = f.readlines()\\n \\n for result in results:\\n stmt = result.get(\\\"stmt\\\")\\n faulty_bit = result.get(\\\"faulty_bit\\\")\\n variant = result.get(\\\"variant\\\")\\n res_val = result.get(\\\"result\\\")\\n \\n # Try to map stmt to line number in .trv file\\n # Based on observation: stmt seems to correlate with line number\\n # Let's assume stmt 0 is after function signature, so roughly line 2+\\n if stmt is not None and stmt >= 0:\\n # Simple mapping: stmt N -> line N+2 (accounting for func line and opening brace)\\n line_index = stmt + 1 # 0-indexed, so stmt 0 = line 1 (0-indexed) = line 2 (1-indexed)\\n if line_index < len(trv_lines):\\n line_content = trv_lines[line_index].rstrip('\\\\n')\\n line_number = line_index + 1\\n print(f\\\" Stmt {stmt} (line {line_number}): {line_content}\\\")\\n print(f\\\" -> Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(f\\\" Stmt {stmt}: (line out of range) Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(f\\\" Stmt: {stmt} (invalid), Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n except Exception as e:\\n print(f\\\" Error reading .trv file: {e}\\\")\\n for result in results:\\n stmt = result.get(\\\"stmt\\\")\\n faulty_bit = result.get(\\\"faulty_bit\\\")\\n variant = result.get(\\\"variant\\\")\\n res_val = result.get(\\\"result\\\")\\n print(f\\\" Stmt {stmt}: Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(f\\\" .trv file not found for test {test_name}\\\")\\n for result in results:\\n stmt = result.get(\\\"stmt\\\")\\n faulty_bit = result.get(\\\"faulty_bit\\\")\\n variant = result.get(\\\"variant\\\")\\n res_val = result.get(\\\"result\\\")\\n print(f\\\" Stmt {stmt}: Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(\\\" No errors found by symex.\\\")\\n print(f\\\"{'='*60}\\\")\\n \\n # Prompt user to press any key\\n print(\\\"\\\\nPress any key to continue (user should manually fix the error points shown above)...\\\")\\n try:\\n # Try to get a single character input\\n import sys\\n import tty\\n import termios\\n fd = sys.stdin.fileno()\\n old_settings = termios.tcgetattr(fd)\\n try:\\n tty.setraw(sys.stdin.fileno())\\n sys.stdin.read(1)\\n finally:\\n termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)\\n except:\\n # Fallback to standard input if termios fails\\n input()\\n\\n plotter = Plotter(self.db)\"}","name":"edit_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":17501,"output_tokens":2851}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"filePath\": \"/home/pbk/school/P10-Master/testing/test\",\n \"oldString\": \" self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n\\n plotter = Plotter(self.db)\",\n \"newString\": \" self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n print(f\\\"\\\\n{'='*60}\\\")\\n print(\\\"ERRORS FOUND BY SYMEX (Symbolic Execution)\\\")\\n print(f\\\"{'='*60}\\\")\\n if symex_results:\\n # Group results by test for better display\\n results_by_test = {}\\n for result in symex_results:\\n test_name = result[\\\"test\\\"]\\n if test_name not in results_by_test:\\n results_by_test[test_name] = []\\n results_by_test[test_name].append(result)\\n \\n for test_name, results in results_by_test.items():\\n print(f\\\"\\\\nTest: {test_name}\\\")\\n print(\\\"-\\\" * 40)\\n # Find the .trv file for this test\\n trv_path = None\\n for test in self.tests:\\n if test[\\\"name\\\"] == test_name:\\n trv_path = test[\\\"path\\\"]\\n break\\n \\n if trv_path and os.path.exists(trv_path):\\n try:\\n with open(trv_path, 'r') as f:\\n trv_lines = f.readlines()\\n \\n for result in results:\\n stmt = result.get(\\\"stmt\\\")\\n faulty_bit = result.get(\\\"faulty_bit\\\")\\n variant = result.get(\\\"variant\\\")\\n res_val = result.get(\\\"result\\\")\\n \\n # Try to map stmt to line number in .trv file\\n # Based on observation: stmt seems to correlate with line number\\n # Let's assume stmt 0 is after function signature, so roughly line 2+\\n if stmt is not None and stmt >= 0:\\n # Simple mapping: stmt N -> line N+2 (accounting for func line and opening brace)\\n line_index = stmt + 1 # 0-indexed, so stmt 0 = line 1 (0-indexed) = line 2 (1-indexed)\\n if line_index < len(trv_lines):\\n line_content = trv_lines[line_index].rstrip('\\\\n')\\n line_number = line_index + 1\\n print(f\\\" Stmt {stmt} (line {line_number}): {line_content}\\\")\\n print(f\\\" -> Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(f\\\" Stmt {stmt}: (line out of range) Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(f\\\" Stmt: {stmt} (invalid), Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n except Exception as e:\\n print(f\\\" Error reading .trv file: {e}\\\")\\n for result in results:\\n stmt = result.get(\\\"stmt\\\")\\n faulty_bit = result.get(\\\"faulty_bit\\\")\\n variant = result.get(\\\"variant\\\")\\n res_val = result.get(\\\"result\\\")\\n print(f\\\" Stmt {stmt}: Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(f\\\" .trv file not found for test {test_name}\\\")\\n for result in results:\\n stmt = result.get(\\\"stmt\\\")\\n faulty_bit = result.get(\\\"faulty_bit\\\")\\n variant = result.get(\\\"variant\\\")\\n res_val = result.get(\\\"result\\\")\\n print(f\\\" Stmt {stmt}: Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}\\\")\\n else:\\n print(\\\" No errors found by symex.\\\")\\n print(f\\\"{'='*60}\\\")\\n \\n # Prompt user to press any key\\n print(\\\"\\\\nPress any key to continue (user should manually fix the error points shown above)...\\\")\\n try:\\n # Try to get a single character input\\n import sys\\n import tty\\n import termios\\n fd = sys.stdin.fileno()\\n old_settings = termios.tcgetattr(fd)\\n try:\\n tty.setraw(sys.stdin.fileno())\\n sys.stdin.read(1)\\n finally:\\n termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)\\n except:\\n # Fallback to standard input if termios fails\\n input()\\n\\n plotter = Plotter(self.db)\",\n \"originalFile\": \"#!/bin/python3\\n\\nimport argparse, os, toml, shutil, subprocess, sys\\nimport matplotlib.pyplot as plt\\nfrom datetime import datetime\\nfrom src.database import ResultsDatabase\\nfrom src.testrunner import TestRunner\\nfrom src.symexrunner import SymexRunner\\nfrom src.plotenator import Plotter\\nfrom rich.console import Console\\n\\n\\nclass Experiment:\\n def __init__(self, args, run_variants, config_path=None):\\n if config_path is None:\\n config_path = os.path.join(os.path.dirname(__file__), \\\"config.toml\\\")\\n\\n with open(config_path, \\\"r\\\") as f:\\n config = toml.load(f)\\n self.args = args\\n self.run_variants = run_variants\\n self.base_dir = os.path.dirname(__file__)\\n self.tests_folder = os.path.join(\\n self.base_dir, config.get(\\\"test_folder\\\", \\\"fissc\\\")\\n )\\n self.artifacts_folder = os.path.join(self.base_dir, \\\"artifacts\\\")\\n self.plot_mode = config.get(\\\"plotmode\\\", \\\"save\\\")\\n self.injection_points = config.get(\\n \\\"injection_points\\\",\\n [\\n \\\"r0\\\",\\n \\\"r1\\\",\\n \\\"r2\\\",\\n \\\"r3\\\",\\n \\\"r4\\\",\\n \\\"r5\\\",\\n \\\"r6\\\",\\n \\\"r7\\\",\\n \\\"r8\\\",\\n \\\"r9\\\",\\n \\\"r10\\\",\\n \\\"r11\\\",\\n \\\"r12\\\",\\n ],\\n )\\n self.default_pass_mode = config.get(\\\"pass_mode\\\", {}).get(\\n \\\"default\\\", \\\"returncode\\\"\\n )\\n self.pass_mode_configs = config.get(\\\"pass_mode\\\", {})\\n self.runner = TestRunner(config_path=args.config)\\n self.tests = self.runner._discover_tests()\\n self.compile_results = None\\n\\n def run(self):\\n\\n # check that the compiler and interpreter is present:\\n self.runner.check_bin()\\n\\n print(f\\\"Setting up...\\\")\\n folder = self.setup()\\n print(f\\\"Created folder: {folder}\\\")\\n\\n print(f\\\"\\\\nCompiling tests...\\\")\\n results = self.runner.compile()\\n print(f\\\"Compiled {len(results)} tests\\\")\\n all_fault_results = None\\n wh_files = sorted(\\n [f for f in os.listdir(self.runner.compiled_folder) if f.endswith(\\\".wh\\\")]\\n )\\n\\n print(\\\"Whiley Files are ready for adding fault injections...\\\")\\n print(\\\"\\\\nRunning symex...\\\")\\n symex_runner = SymexRunner(config_path=self.args.config)\\n symex_runner.artifacts_folder = self.runner.compiled_folder\\n symex_results = symex_runner.run(self.run_variants)\\n\\n self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\"\\\\nRunning Interpreter Guided...\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\"\\\\nRunning Interpreter Exhaustively...\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\"\\\\nSkipping test run (--skip-run)\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n\\n plotter = Plotter(self.db)\\n plots = plotter.create_plots()\\n\\n\\n plots_dir = os.path.join(self.run_folder, \\\"plots\\\")\\n os.makedirs(plots_dir, exist_ok=True)\\n\\n\\n def save_plot(name, fig):\\n \\\"\\\"\\\"Handles the saving of the figure to a specific directory.\\\"\\\"\\\"\\n path = os.path.join(plots_dir, f\\\"{name}.svg\\\")\\n fig.savefig(path, format=\\\"svg\\\", bbox_inches=\\\"tight\\\")\\n\\n # only these remain as plots\\n if self.plot_mode == \\\"save\\\" or self.plot_mode == \\\"both\\\":\\n save_plot(\\\"exhaustive\\\", plots[\\\"exhaustive\\\"])\\n save_plot(\\\"guided\\\", plots[\\\"guided\\\"])\\n\\n if self.plot_mode == \\\"show\\\" or self.plot_mode == \\\"both\\\":\\n plt.show()\\n for fig in plots.values():\\n plt.close(fig)\\n\\n # symex exported as CSV instead of plot\\n symex_csv_path = os.path.join(plots_dir, \\\"symex_results.csv\\\")\\n plotter.export_symex_csv(symex_csv_path)\\n\\n self.db.close()\\n \\n zip_base = os.path.join(self.artifacts_folder, self.run_name)\\n shutil.make_archive(\\n zip_base, \\\"zip\\\", self.artifacts_folder, self.run_folder_name\\n )\\n shutil.rmtree(self.run_folder)\\n\\n print(f\\\"\\\\nArtifacts saved to: {zip_base}.zip\\\")\\n\\n def setup(self):\\n timestamp = datetime.now().strftime(\\\"%Y%m%d_%H%M%S\\\")\\n run_num = self._get_next_run_number()\\n self.run_name = f\\\"{run_num}-{timestamp}\\\"\\n self.run_folder_name = f\\\"test_results\\\"\\n self.run_folder = os.path.join(self.artifacts_folder, self.run_folder_name)\\n self.run_tests_folder = os.path.join(self.run_folder, \\\"tests\\\")\\n self.runner.compiled_folder = os.path.join(\\n self.run_folder, \\\"compiled_test_files\\\"\\n )\\n db_path = os.path.join(self.run_folder, \\\"results.db\\\")\\n\\n os.makedirs(self.run_folder, exist_ok=True)\\n os.makedirs(self.run_tests_folder, exist_ok=True)\\n os.makedirs(self.runner.compiled_folder, exist_ok=True)\\n\\n for f in os.listdir(self.tests_folder):\\n if f.endswith(\\\".trv\\\") or f.endswith(\\\".trv.output\\\") or f.endswith(\\\".wh\\\"):\\n shutil.copy(os.path.join(self.tests_folder, f), self.run_tests_folder)\\n\\n self.db = ResultsDatabase(db_path)\\n self.run_id = self.db.create_run(run_num, self.tests_folder)\\n\\n return self.run_folder\\n\\n def _get_next_run_number(self):\\n if not os.path.exists(self.artifacts_folder):\\n return 1\\n\\n max_num = 0\\n for item in os.listdir(self.artifacts_folder):\\n base_name = item[:-4] if item.endswith(\\\".zip\\\") else item\\n if os.path.isdir(\\n os.path.join(self.artifacts_folder, item)\\n ) or item.endswith(\\\".zip\\\"):\\n parts = base_name.split(\\\"-\\\", 1)\\n if len(parts) == 2 and parts[0].isdigit():\\n max_num = max(max_num, int(parts[0]))\\n return max_num + 1\\n\\n\\ndef run_setup(target=\\\"all\\\", force=False):\\n script_dir = os.path.dirname(os.path.abspath(__file__))\\n setup_module = os.path.join(script_dir, \\\"src\\\", \\\"setup.py\\\")\\n\\n cmd = [sys.executable, setup_module, \\\"--target\\\", target]\\n if force:\\n cmd.append(\\\"-f\\\")\\n\\n result = subprocess.run(cmd)\\n return result.returncode\\n\\n\\ndef get_missing_binaries():\\n script_dir = os.path.dirname(os.path.abspath(__file__))\\n bin_dir = os.path.join(script_dir, \\\"bin\\\")\\n required_bins = [\\\"minimc\\\", \\\"compiler\\\", \\\"interpreter\\\"]\\n missing = []\\n for bin_name in required_bins:\\n if not os.path.isfile(os.path.join(bin_dir, bin_name)):\\n missing.append(bin_name)\\n return missing\\n\\n\\nif __name__ == \\\"__main__\\\":\\n parser = argparse.ArgumentParser()\\n parser.add_argument(\\\"--config\\\", type=str, default=None, help=\\\"Path to config.toml\\\")\\n parser.add_argument(\\n \\\"--setup\\\",\\n type=str,\\n nargs=\\\"?\\\",\\n const=\\\"all\\\",\\n default=None,\\n metavar=\\\"TARGETS\\\",\\n help=\\\"Only run setup (build dependencies), then quit. Comma-separated: compiler, interpreter, minimc, or all\\\",\\n )\\n parser.add_argument(\\n \\\"--skip-run\\\",\\n action=\\\"store_true\\\",\\n help=\\\"Only setup and compile, skip running tests\\\",\\n )\\n parser.add_argument(\\n \\\"--limit\\\",\\n type=int,\\n default=None,\\n help=\\\"Limit number of injection points to test\\\",\\n )\\n parser.add_argument(\\n \\\"--nohard\\\",\\n action=\\\"store_true\\\",\\n help=\\\"Skip hardened version when running tests\\\",\\n )\\n parser.add_argument(\\\"--hard\\\", action=\\\"store_true\\\", help=\\\"Only run hardened version\\\")\\n args, unknown = parser.parse_known_args()\\n\\n force = \\\"-f\\\" in unknown or \\\"--reinit\\\" in unknown\\n\\n if args.setup:\\n exit(run_setup(target=args.setup, force=force))\\n\\n missing_bins = get_missing_binaries()\\n if missing_bins:\\n print(f\\\"\\\\033[91mError: Binaries not found: {', '.join(missing_bins)}\\\\033[0m\\\")\\n if len(missing_bins) > 1:\\n print(\\\"Run 'test --setup' to compile the binaries\\\")\\n exit(1)\\n print(\\\"Run 'test --setup' to compile the binary\\\")\\n exit(1)\\n\\n\\n if args.nohard and args.hard:\\n print(\\\"Error: Cannot use both --nohard and --hard\\\")\\n exit()\\n\\n if args.hard:\\n run_variants = \\\"hard\\\"\\n elif args.nohard:\\n run_variants = \\\"normal\\\"\\n else:\\n run_variants = \\\"both\\\"\\n exp = Experiment(args, run_variants)\\n exp.run()\\n\",\n \"structuredPatch\": [\n {\n \"oldStart\": 1,\n \"oldLines\": 253,\n \"newStart\": 1,\n \"newLines\": 338,\n \"lines\": [\n \"-#!/bin/python3\",\n \"-\",\n \"-import argparse, os, toml, shutil, subprocess, sys\",\n \"-import matplotlib.pyplot as plt\",\n \"-from datetime import datetime\",\n \"-from src.database import ResultsDatabase\",\n \"-from src.testrunner import TestRunner\",\n \"-from src.symexrunner import SymexRunner\",\n \"-from src.plotenator import Plotter\",\n \"-from rich.console import Console\",\n \"-\",\n \"-\",\n \"-class Experiment:\",\n \"- def __init__(self, args, run_variants, config_path=None):\",\n \"- if config_path is None:\",\n \"- config_path = os.path.join(os.path.dirname(__file__), \\\"config.toml\\\")\",\n \"-\",\n \"- with open(config_path, \\\"r\\\") as f:\",\n \"- config = toml.load(f)\",\n \"- self.args = args\",\n \"- self.run_variants = run_variants\",\n \"- self.base_dir = os.path.dirname(__file__)\",\n \"- self.tests_folder = os.path.join(\",\n \"- self.base_dir, config.get(\\\"tes… [truncated for session JSONL]","tool_name":"edit_file","tool_use_id":"chatcmpl-tool-a4a7992623b11909","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-ba43ce9dfbd5269a","input":"{\"command\": \"cd /home/pbk/school/P10-Master/testing && python3 -m py_compile test\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":29814,"output_tokens":91}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"\",\n \"stderr\": \"\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": null,\n \"noOutputExpected\": true,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-ba43ce9dfbd5269a","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-8564a22bab629018","input":"{\"command\": \"cd /home/pbk/school/P10-Master/testing && python3 test --help\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":30146,"output_tokens":85}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"\",\n \"stderr\": \"Traceback (most recent call last):\\n File \\\"/home/pbk/school/P10-Master/testing/test\\\", line 3, in \\n import argparse, os, toml, shutil, subprocess, sys\\nModuleNotFoundError: No module named 'toml'\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": \"exit_code:1\",\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-8564a22bab629018","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-9fb37dbb03b19642","input":"{\"command\": \"pip install toml\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":30546,"output_tokens":43}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"Defaulting to user installation because normal site-packages is not writeable\\nCollecting toml\\n Using cached toml-0.10.2-py2.py3-none-any.whl.metadata (7.1 kB)\\nUsing cached toml-0.10.2-py2.py3-none-any.whl (16 kB)\\nInstalling collected packages: toml\\nSuccessfully installed toml-0.10.2\\n\",\n \"stderr\": \"WARNING: Cache entry deserialization failed, entry ignored\\nWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": null,\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-9fb37dbb03b19642","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-a7062397854f1027","input":"{\"command\": \"ls -la /home/pbk/school/P10-Master/testing/bin/\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":31047,"output_tokens":280}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"total 12216\\ndrwxr-xr-x 1 root root 70 May 13 10:41 .\\ndrwxr-xr-x 1 root root 268 May 18 13:26 ..\\n-rwxr-xr-x 1 root root 1180120 May 13 11:55 compiler\\n-rwxr-xr-x 1 root root 1199096 May 1 10:54 interpreter\\n-rwxr-xr-x 1 root root 8823752 May 13 10:41 minimc\\n-rw-r--r-- 1 root root 1295818 May 1 13:35 minimc.zip\\n\",\n \"stderr\": \"\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": null,\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-a7062397854f1027","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-926fcaaab336faae","input":"{\"command\": \"cd /home/pbk/school/P10-Master/testing && python3 test --skip-run\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":31561,"output_tokens":74}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"\",\n \"stderr\": \"Traceback (most recent call last):\\n File \\\"/home/pbk/school/P10-Master/testing/test\\\", line 4, in \\n import matplotlib.pyplot as plt\\nModuleNotFoundError: No module named 'matplotlib'\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": \"exit_code:1\",\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-926fcaaab336faae","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-99a4ba50353c7816","input":"{\"command\": \"pip install matplotlib rich\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":31953,"output_tokens":86}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"Defaulting to user installation because normal site-packages is not writeable\\nCollecting matplotlib\\n Using cached matplotlib-3.10.9-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (52 kB)\\nCollecting rich\\n Using cached rich-15.0.0-py3-none-any.whl.metadata (18 kB)\\nCollecting contourpy>=1.0.1 (from matplotlib)\\n Using cached contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (5.5 kB)\\nCollecting cycler>=0.10 (from matplotlib)\\n Using cached cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\\nCollecting fonttools>=4.22.0 (from matplotlib)\\n Downloading fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (118 kB)\\nCollecting kiwisolver>=1.3.1 (from matplotlib)\\n Using cached kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (5.1 kB)\\nRequirement already satisfied: numpy>=1.23 in /usr/lib/python3.14/site-packages (from matplotlib) (2.4.4)\\nRequirement already satisfied: packaging>=20.0 in /usr/lib/python3.14/site-packages (from matplotlib) (26.2)\\nRequirement already satisfied: pillow>=8 in /usr/lib/python3.14/site-packages (from matplotlib) (12.2.0)\\nCollecting pyparsing>=3 (from matplotlib)\\n Using cached pyparsing-3.3.2-py3-none-any.whl.metadata (5.8 kB)\\nRequirement already satisfied: python-dateutil>=2.7 in /usr/lib/python3.14/site-packages (from matplotlib) (2.9.0)\\nCollecting markdown-it-py>=2.2.0 (from rich)\\n Downloading markdown_it_py-4.2.0-py3-none-any.whl.metadata (7.4 kB)\\nRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/lib/python3.14/site-packages (from rich) (2.20.0)\\nCollecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich)\\n Using cached mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)\\nRequirement already satisfied: six>=1.5 in /usr/lib/python3.14/site-packages (from python-dateutil>=2.7->matplotlib) (1.17.0)\\nUsing cached matplotlib-3.10.9-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (9.9 MB)\\nUsing cached rich-15.0.0-py3-none-any.whl (310 kB)\\nUsing cached contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (363 kB)\\nUsing cached cycler-0.12.1-py3-none-any.whl (8.3 kB)\\nDownloading fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (4.9 MB)\\n ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.9/4.9 MB 6.3 MB/s 0:00:00\\nUsing cached kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (1.5 MB)\\nDownloading markdown_it_py-4.2.0-py3-none-any.whl (91 kB)\\nUsing cached mdurl-0.1.2-py3-none-any.whl (10.0 kB)\\nUsing cached pyparsing-3.3.2-py3-none-any.whl (122 kB)\\nInstalling collected packages: pyparsing, mdurl, kiwisolver, fonttools, cycler, contourpy, matplotlib, markdown-it-py, rich\\n\\nSuccessfully installed contourpy-1.3.3 cycler-0.12.1 fonttools-4.63.0 kiwisolver-1.5.0 markdown-it-py-4.2.0 matplotlib-3.10.9 mdurl-0.1.2 pyparsing-3.3.2 rich-15.0.0\\n\",\n \"stderr\": \"WARNING: Cache entry deserialization failed, entry ignored\\nWARNING: Cache entry deserialization failed, entry ignored\\nWARNING: Cache entry deserialization failed, entry ignored\\nWARNING: Cache entry deserialization failed, entry ignored\\n WARNING: The scripts fonttools, pyftmerge, pyftsubset and ttx are installed in '/home/pbk/school/P10-Master/testing/.sandbox-home/.local/bin' which is not on PATH.\\n Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\\n WARNING: The script markdown-it is installed in '/home/pbk/school/P10-Master/testing/.sandbox-home/.local/bin' which is not on PATH.\\n Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\\nWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": null,\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-99a4ba50353c7816","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-9aca086999918eea","input":"{\"command\": \"cd /home/pbk/school/P10-Master/testing && python3 test --skip-run\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":33863,"output_tokens":110}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"\",\n \"stderr\": \"Traceback (most recent call last):\\n File \\\"/home/pbk/school/P10-Master/testing/test\\\", line 6, in \\n from src.database import ResultsDatabase\\n File \\\"/home/pbk/school/P10-Master/testing/src/database.py\\\", line 4, in \\n import pandas as pd\\nModuleNotFoundError: No module named 'pandas'\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": \"exit_code:1\",\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-9aca086999918eea","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-b878ac6233c3dc18","input":"{\"command\": \"pip install pandas\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":34291,"output_tokens":53}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"Defaulting to user installation because normal site-packages is not writeable\\nCollecting pandas\\n Downloading pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (79 kB)\\nRequirement already satisfied: numpy>=2.3.3 in /usr/lib/python3.14/site-packages (from pandas) (2.4.4)\\nRequirement already satisfied: python-dateutil>=2.8.2 in /usr/lib/python3.14/site-packages (from pandas) (2.9.0)\\nRequirement already satisfied: six>=1.5 in /usr/lib/python3.14/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\\nDownloading pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (10.9 MB)\\n ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.9/10.9 MB 5.4 MB/s 0:00:02\\nInstalling collected packages: pandas\\nSuccessfully installed pandas-3.0.3\\n\",\n \"stderr\": \"WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": null,\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-b878ac6233c3dc18","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-99a01561df328e85","input":"{\"command\": \"cd /home/pbk/school/P10-Master/testing && timeout 30 python3 test --skip-run\"}","name":"bash","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":35055,"output_tokens":99}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"stdout\": \"Setting up...\\nCreated folder: /home/pbk/school/P10-Master/testing/artifacts/test_results\\n\\nCompiling tests...\\nCompiled 1 tests\\nWhiley Files are ready for adding fault injections...\\n\\nRunning symex...\\nRunning MiniMC on pinny.hard.wh\\nRunning MiniMC on pinny.wh\\n\\nSkipping test run (--skip-run)\\n\",\n \"stderr\": \"Traceback (most recent call last):\\n File \\\"/home/pbk/school/P10-Master/testing/test\\\", line 338, in \\n exp.run()\\n ~~~~~~~^^\\n File \\\"/home/pbk/school/P10-Master/testing/test\\\", line 89, in run\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n ^^^^^^^^^^^^^^^^^^^^^^^^\\nUnboundLocalError: cannot access local variable 'exhaustive_fault_results' where it is not associated with a value\\n\",\n \"rawOutputPath\": null,\n \"interrupted\": false,\n \"isImage\": null,\n \"backgroundTaskId\": null,\n \"backgroundedByUser\": null,\n \"assistantAutoBackgrounded\": null,\n \"dangerouslyDisableSandbox\": null,\n \"returnCodeInterpretation\": \"exit_code:1\",\n \"noOutputExpected\": false,\n \"structuredContent\": null,\n \"persistedOutputPath\": null,\n \"persistedOutputSize\": null,\n \"sandboxStatus\": {\n \"enabled\": true,\n \"requested\": {\n \"enabled\": true,\n \"namespace_restrictions\": true,\n \"network_isolation\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"allowed_mounts\": []\n },\n \"supported\": true,\n \"active\": true,\n \"namespace_supported\": true,\n \"namespace_active\": true,\n \"network_supported\": true,\n \"network_active\": false,\n \"filesystem_mode\": \"workspace-only\",\n \"filesystem_active\": true,\n \"allowed_mounts\": [],\n \"in_container\": false,\n \"container_markers\": [],\n \"fallback_reason\": null\n }\n}","tool_name":"bash","tool_use_id":"chatcmpl-tool-99a01561df328e85","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-a53762df5c5f557b","input":"{\"path\": \"test\", \"offset\": 70, \"limit\": 30}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":35620,"output_tokens":130}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/test\",\n \"content\": \" print(\\\"Whiley Files are ready for adding fault injections...\\\")\\n print(\\\"\\\\nRunning symex...\\\")\\n symex_runner = SymexRunner(config_path=self.args.config)\\n symex_runner.artifacts_folder = self.runner.compiled_folder\\n symex_results = symex_runner.run(self.run_variants)\\n\\n self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\"\\\\nRunning Interpreter Guided...\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\"\\\\nRunning Interpreter Exhaustively...\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\"\\\\nSkipping test run (--skip-run)\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n print(f\\\"\\\\n{'='*60}\\\")\\n print(\\\"ERRORS FOUND BY SYMEX (Symbolic Execution)\\\")\\n print(f\\\"{'='*60}\\\")\\n if symex_results:\\n # Group results by test for better display\\n results_by_test = {}\\n for result in symex_results:\",\n \"numLines\": 30,\n \"startLine\": 71,\n \"totalLines\": 338\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-a53762df5c5f557b","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-a728e8078b9de970","input":"{\"path\": \"test\", \"old_string\": \" self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Guided...\\\\\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Exhaustively...\\\\\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\\\\\"\\\\\\\\nSkipping test run (--skip-run)\\\\\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\", \"new_string\": \" self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n # Initialize variables to avoid UnboundLocalError when skip_run is True\\n guided_fault_results = []\\n exhaustive_fault_results = []\\n\\n if not self.args.skip_run:\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Guided...\\\\\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Exhaustively...\\\\\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\\\\\"\\\\\\\\nSkipping test run (--skip-run)\\\\\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\"}","name":"edit_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":36184,"output_tokens":498}},"type":"message"} +{"message":{"blocks":[{"is_error":true,"output":"old_string not found in file","tool_name":"edit_file","tool_use_id":"chatcmpl-tool-a728e8078b9de970","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-9a0f07dd9962aa9a","input":"{\"path\": \"test\", \"offset\": 70, \"limit\": 30}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":36629,"output_tokens":67}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/test\",\n \"content\": \" print(\\\"Whiley Files are ready for adding fault injections...\\\")\\n print(\\\"\\\\nRunning symex...\\\")\\n symex_runner = SymexRunner(config_path=self.args.config)\\n symex_runner.artifacts_folder = self.runner.compiled_folder\\n symex_results = symex_runner.run(self.run_variants)\\n\\n self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\"\\\\nRunning Interpreter Guided...\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\"\\\\nRunning Interpreter Exhaustively...\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\"\\\\nSkipping test run (--skip-run)\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\\n\\n # HERE: Place here, a print that displays the errors that was found when running symex, and figure out how to show that as lines in the .trv files.\\n # then prompt the user to press any key, (the user is then to manually fix those errors points that symex found).\\n print(f\\\"\\\\n{'='*60}\\\")\\n print(\\\"ERRORS FOUND BY SYMEX (Symbolic Execution)\\\")\\n print(f\\\"{'='*60}\\\")\\n if symex_results:\\n # Group results by test for better display\\n results_by_test = {}\\n for result in symex_results:\",\n \"numLines\": 30,\n \"startLine\": 71,\n \"totalLines\": 338\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-9a0f07dd9962aa9a","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-9093b2a035dcc86a","input":"{\"path\": \"test\", \"limit\": 10, \"offset\": 75}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":37193,"output_tokens":63}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/test\",\n \"content\": \"\\n self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\"\\\\nRunning Interpreter Guided...\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\"\\\\nRunning Interpreter Exhaustively...\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\",\n \"numLines\": 10,\n \"startLine\": 76,\n \"totalLines\": 338\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-9093b2a035dcc86a","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-9a05c24cea163db8","input":"{\"limit\": 15, \"offset\": 75, \"path\": \"test\"}","name":"read_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":37456,"output_tokens":62}},"type":"message"} +{"message":{"blocks":[{"is_error":false,"output":"{\n \"type\": \"text\",\n \"file\": {\n \"filePath\": \"/home/pbk/school/P10-Master/testing/test\",\n \"content\": \"\\n self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\"\\\\nRunning Interpreter Guided...\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\"\\\\nRunning Interpreter Exhaustively...\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\"\\\\nSkipping test run (--skip-run)\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\",\n \"numLines\": 15,\n \"startLine\": 76,\n \"totalLines\": 338\n }\n}","tool_name":"read_file","tool_use_id":"chatcmpl-tool-9a05c24cea163db8","type":"tool_result"}],"role":"tool"},"type":"message"} +{"message":{"blocks":[{"id":"chatcmpl-tool-aea1b7a467d686fd","input":"{\"new_string\": \" self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n # Initialize variables to avoid UnboundLocalError when skip_run is True\\n guided_fault_results = []\\n exhaustive_fault_results = []\\n\\n if not self.args.skip_run:\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Guided...\\\\\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Exhaustively...\\\\\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\\\\\"\\\\\\\\nSkipping test run (--skip-run)\\\\\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\", \"old_string\": \" self.db.insert_symex_results_batch(self.run_id, symex_results)\\n\\n if not self.args.skip_run:\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Guided...\\\\\\\")\\n guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results)\\n print(f\\\\\\\"\\\\\\\\nRunning Interpreter Exhaustively...\\\\\\\")\\n exhaustive_fault_results = self.runner.run_tests(\\n limit=self.args.limit, run_variants=run_variants\\n )\\n else:\\n print(f\\\\\\\"\\\\\\\\nSkipping test run (--skip-run)\\\\\\\")\\n\\n self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results)\\n self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True)\", \"path\": \"test\"}","name":"edit_file","type":"tool_use"}],"role":"assistant","usage":{"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"input_tokens":37779,"output_tokens":449}},"type":"message"} +{"message":{"blocks":[{"is_error":true,"output":"old_string not found in file","tool_name":"edit_file","tool_use_id":"chatcmpl-tool-aea1b7a467d686fd","type":"tool_result"}],"role":"tool"},"type":"message"} diff --git a/testing/__pycache__/testcpython-314.pyc b/testing/__pycache__/testcpython-314.pyc new file mode 100644 index 0000000..00930a6 Binary files /dev/null and b/testing/__pycache__/testcpython-314.pyc differ diff --git a/testing/config.toml b/testing/config.toml index 9d7d199..0bbcc2b 100644 --- a/testing/config.toml +++ b/testing/config.toml @@ -1,11 +1,7 @@ test_folder = "pinny" -plotmode = "both" # show, save or both +plotmode = "save" # show, save or both -injection_points = [ - "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", - "r8", "r9", "r10", "r11", "r12", "pc=*", - "cpsr_n" -] +injection_points = [] [pass_mode] default = "returncode" diff --git a/testing/run_tests b/testing/run_tests new file mode 100755 index 0000000..bdcc20e --- /dev/null +++ b/testing/run_tests @@ -0,0 +1,529 @@ +#!/bin/python3 + +import argparse, os, toml, shutil, subprocess, sys +import matplotlib.pyplot as plt +from datetime import datetime +from src.database import ResultsDatabase +from src.testrunner import TestRunner +from src.symexrunner import SymexRunner +from src.plotenator import Plotter +from rich.console import Console + + +class Experiment: + def __init__(self, args, run_variants, config_path=None): + if config_path is None: + config_path = os.path.join(os.path.dirname(__file__), "config.toml") + + with open(config_path, "r") as f: + config = toml.load(f) + self.args = args + self.run_variants = run_variants + self.base_dir = os.path.dirname(__file__) + self.tests_folder = os.path.join( + self.base_dir, config.get("test_folder", "fissc") + ) + self.artifacts_folder = os.path.join(self.base_dir, "artifacts") + self.plot_mode = config.get("plotmode", "save") + self.injection_points = config.get( + "injection_points", + [ + "r0", + "r1", + "r2", + "r3", + "r4", + "r5", + "r6", + "r7", + "r8", + "r9", + "r10", + "r11", + "r12", + ], + ) + self.default_pass_mode = config.get("pass_mode", {}).get( + "default", "returncode" + ) + self.pass_mode_configs = config.get("pass_mode", {}) + self.runner = TestRunner(config_path=args.config) + self.tests = self.runner._discover_tests() + self.compile_results = None + + def _build_stmt_to_trv_mapping(self, wh_path, trv_path): + """Build a mapping from stmt numbers to .trv file lines.""" + import re + stmt_to_expr = {} + + if not os.path.exists(wh_path) or not os.path.exists(trv_path): + return stmt_to_expr + + with open(wh_path, 'r') as f: + wh_lines = f.readlines() + + for i, line in enumerate(wh_lines): + match = re.search(r'if\s*\(stmt\s*==\s*\((\d+)\s+as\s+ui32\)\)', line) + if match: + stmt_num = int(match.group(1)) + if i > 0: + prev_line = wh_lines[i-1].strip().rstrip(';') + expr_match = re.match(r'(\w+)\s*=\s*(.+)', prev_line) + if expr_match: + var_name = expr_match.group(1) + rhs = expr_match.group(2).strip() + rhs_normalized = re.sub(r'\s*\(?\d+\s+as\s+ui32\)?', lambda m: re.search(r'\d+', m.group()).group(), rhs) + stmt_to_expr[stmt_num] = (var_name, rhs_normalized) + + with open(trv_path, 'r') as f: + trv_lines = f.readlines() + + stmt_to_trv_line = {} + used_trv_lines = set() + + for stmt_num, (var_name, rhs_normalized) in stmt_to_expr.items(): + best_match = None + for line_idx, trv_line in enumerate(trv_lines): + if line_idx in used_trv_lines: + continue + stripped = trv_line.strip().rstrip(';') + + trv_expr_match = re.match(r'(?:let\s+)?(\w+)\s*(?::\s*\w+\s*)?=\s*(.+)', stripped) + if trv_expr_match: + trv_var = trv_expr_match.group(1) + trv_rhs = trv_expr_match.group(2).strip() + + if trv_var == var_name: + if rhs_normalized == trv_rhs: + best_match = (line_idx + 1, stripped) + break + elif best_match is None and re.search(re.escape(var_name), trv_rhs): + best_match = (line_idx + 1, stripped) + + if best_match: + stmt_to_trv_line[stmt_num] = best_match + used_trv_lines.add(best_match[0] - 1) + + return stmt_to_trv_line + + def _run_edited_tests(self, edited_trv_path, test_name): + """Compile and run symex + fault injection on an edited .trv file.""" + edited_folder = os.path.join(self.run_folder, "edited_test_files") + os.makedirs(edited_folder, exist_ok=True) + + print(f"\n{'='*60}") + print(f"RE-RUNNING ANALYSIS ON EDITED FILE: {os.path.basename(edited_trv_path)}") + print(f"{'='*60}") + + edited_test_name = os.path.splitext(os.path.basename(edited_trv_path))[0] + + edited_test = { + "name": edited_test_name, + "path": edited_trv_path, + "expected_output": "", + "pass_mode": {"mode": "returncode", "expected_pass": 0, "expected_fail": 1}, + } + + orig_tests = self.runner.tests + orig_compile_results = self.runner.compile_results + orig_compiled_folder = self.runner.compiled_folder + + self.runner.tests = [edited_test] + self.runner.compile_results = None + self.runner.compiled_folder = edited_folder + + print(f"\nCompiling edited test...") + compile_results = self.runner.compile() + for res in compile_results: + if res["normal"] and res["normal"]["success"]: + print(f" Compiled successfully: {res['normal']['asm_path']}") + else: + print(f" Compilation failed!") + if res["normal"] and res["normal"]["stderr"]: + print(f" stderr: {res['normal']['stderr']}") + self.runner.tests = orig_tests + self.runner.compile_results = orig_compile_results + self.runner.compiled_folder = orig_compiled_folder + return + + print(f"\nRunning symex on edited file...") + symex_runner = SymexRunner(config_path=self.args.config) + symex_runner.artifacts_folder = edited_folder + edited_symex_results = symex_runner.run("normal") + + self.db.insert_symex_results_batch(self.run_id, edited_symex_results, edited=True) + print(edited_symex_results) + + print(f"\nRunning Interpreter Guided on edited file...") + guided_results = self.runner.run_tests(limit=self.args.limit, run_variants="normal", minimc_res=edited_symex_results) + + print(f"\nRunning Interpreter Exhaustively on edited file...") + self.runner.compile_results = None + exhaustive_results = self.runner.run_tests(limit=self.args.limit, run_variants="normal") + + self.db.insert_interpreter_results_batch(self.run_id, exhaustive_results, edited=True) + self.db.insert_interpreter_results_batch(self.run_id, guided_results, True, edited=True) + + print(f"\nDisplaying errors found by symex on edited file (normal variant only)...") + print(f"{'='*60}") + if edited_symex_results: + normal_results = [r for r in edited_symex_results if r.get("variant") == "Normal"] + if normal_results: + results_by_test = {} + for result in normal_results: + tname = result["test"] + if tname not in results_by_test: + results_by_test[tname] = [] + results_by_test[tname].append(result) + + for tname, results in results_by_test.items(): + print(f"\nTest: {tname}") + print("-" * 40) + for result in results: + stmt = result.get("stmt") + faulty_bit = result.get("faulty_bit") + variant = result.get("variant") + res_val = result.get("result") + print(f" Stmt {stmt}: Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}") + else: + print(" No errors found by symex for normal variant.") + else: + print(" No errors found by symex.") + print(f"{'='*60}") + + self.runner.tests = orig_tests + self.runner.compile_results = orig_compile_results + self.runner.compiled_folder = orig_compiled_folder + + return edited_symex_results, guided_results, exhaustive_results + + def run(self): + + # check that the compiler and interpreter is present: + self.runner.check_bin() + + print(f"Setting up...") + folder = self.setup() + print(f"Created folder: {folder}") + + print(f"\nCompiling tests...") + results = self.runner.compile() + print(f"Compiled {len(results)} tests") + all_fault_results = None + wh_files = sorted( + [f for f in os.listdir(self.runner.compiled_folder) if f.endswith(".wh")] + ) + + print("Whiley Files are ready for adding fault injections...") + print("\nRunning symex...") + symex_runner = SymexRunner(config_path=self.args.config) + symex_runner.artifacts_folder = self.runner.compiled_folder + symex_results = symex_runner.run(self.run_variants) + + self.db.insert_symex_results_batch(self.run_id, symex_results) + + if not self.args.skip_run: + print(f"\nRunning Interpreter Guided...") + guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results) + print(f"\nRunning Interpreter Exhaustively...") + exhaustive_fault_results = self.runner.run_tests( + limit=self.args.limit, run_variants=run_variants + ) + self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results) + self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True) + else: + print(f"\nSkipping test run (--skip-run)") + + + # Display errors found by symex for normal variant only + print(f"\n{'='*60}") + print("ERRORS FOUND BY SYMEX (Symbolic Execution) - Normal Variant Only") + print(f"{'='*60}") + trv_files_to_edit = set() + if symex_results: + normal_results = [r for r in symex_results if r.get("variant") == "Normal"] + if normal_results: + results_by_test = {} + for result in normal_results: + test_name = result["test"] + if test_name not in results_by_test: + results_by_test[test_name] = [] + results_by_test[test_name].append(result) + + for test_name, results in results_by_test.items(): + print(f"\nTest: {test_name}") + print("-" * 40) + + trv_path = None + wh_path = None + for test in self.tests: + if test["name"] == test_name: + trv_path = test["path"] + break + + if trv_path: + test_base = os.path.splitext(os.path.basename(trv_path))[0] + wh_path = os.path.join(self.runner.compiled_folder, f"{test_base}.wh") + artifact_trv = os.path.join(self.run_tests_folder, f"{test_base}.trv") + if os.path.exists(artifact_trv): + trv_files_to_edit.add(artifact_trv) + + stmt_mapping = self._build_stmt_to_trv_mapping(wh_path, trv_path) if trv_path and wh_path else {} + + if trv_path and os.path.exists(trv_path): + for result in results: + stmt = result.get("stmt") + faulty_bit = result.get("faulty_bit") + variant = result.get("variant") + res_val = result.get("result") + + if stmt in stmt_mapping: + line_number, line_content = stmt_mapping[stmt] + print(f" Stmt {stmt} (line {line_number}): {line_content}") + print(f" -> Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}") + else: + print(f" Stmt {stmt}: (could not map to source) Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}") + else: + print(f" .trv file not found for test {test_name}") + for result in results: + stmt = result.get("stmt") + faulty_bit = result.get("faulty_bit") + variant = result.get("variant") + res_val = result.get("result") + print(f" Stmt {stmt}: Faulty bit: {faulty_bit}, Variant: {variant}, Result: {res_val}") + else: + print(" No errors found by symex for normal variant.") + else: + print(" No errors found by symex.") + print(f"{'='*60}") + + if trv_files_to_edit: + editor = os.environ.get("EDITOR", "nano") + + known_terminals = ["foot", "alacritty", "kitty", "gnome-terminal", "xterm", "konsole", "ghostty"] + term_cmd = None + for t in known_terminals: + if shutil.which(t): + term_cmd = t + break + + edited_files = [] + for trv_file in sorted(trv_files_to_edit): + edited_path = trv_file.replace(".trv", ".edited.trv") + shutil.copy2(trv_file, edited_path) + edited_files.append(edited_path) + + while True: + editor_procs = [] + for edited_file in edited_files: + print(f"\nOpening {edited_file} in {editor} (new terminal window)...") + if term_cmd: + if term_cmd == "gnome-terminal": + p = subprocess.Popen(["gnome-terminal", "--wait", "--", editor, edited_file], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + elif term_cmd == "xterm": + p = subprocess.Popen(["xterm", "-e", editor, edited_file], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + elif term_cmd == "konsole": + p = subprocess.Popen(["konsole", "-e", editor, edited_file], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + elif term_cmd in ("alacritty", "kitty", "foot", "ghostty"): + p = subprocess.Popen([term_cmd, "-e", editor, edited_file], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + editor_procs.append(p) + else: + print(f" Warning: No terminal emulator found, opening in current terminal") + subprocess.run([editor, edited_file]) + + for p in editor_procs: + p.wait() + + while True: + try: + response = input("\nHave you made the edits you wanted? (y/n): ").strip().lower() + if response in ("y", "n"): + break + except EOFError: + response = "y" + break + + if response == "y": + break + + print(f"\n{'='*60}") + print("Compiling and re-running analysis on edited files...") + print(f"{'='*60}") + + for edited_file in edited_files: + test_base = os.path.splitext(os.path.basename(edited_file))[0].replace(".edited", "") + self._run_edited_tests(edited_file, test_base) + + + + plotter = Plotter(self.db) + plots = plotter.create_plots() + + + plots_dir = os.path.join(self.run_folder, "plots") + os.makedirs(plots_dir, exist_ok=True) + + + def save_plot(name, fig): + """Handles the saving of the figure to a specific directory.""" + path = os.path.join(plots_dir, f"{name}.svg") + fig.savefig(path, format="svg", bbox_inches="tight") + + # only these remain as plots + if self.plot_mode == "save" or self.plot_mode == "both": + print("Saving Plots...") + save_plot("exhaustive", plots["exhaustive"]) + save_plot("exhaustive_edited", plots["exhaustive_edited"]) + save_plot("guided", plots["guided"]) + save_plot("guided_edited", plots["guided_edited"]) + save_plot("exhaustive_fault_types_normal", plots["exhaustive_fault_types_normal"]) + save_plot("exhaustive_fault_types_hard", plots["exhaustive_fault_types_hard"]) + save_plot("exhaustive_fault_types_normal_edited", plots["exhaustive_fault_types_normal_edited"]) + save_plot("guided_fault_types_normal", plots["guided_fault_types_normal"]) + save_plot("guided_fault_types_hard", plots["guided_fault_types_hard"]) + save_plot("guided_fault_types_normal_edited", plots["guided_fault_types_normal_edited"]) + + if self.plot_mode == "show" or self.plot_mode == "both": + print("Showing Plots...") + plt.show() + for fig in plots.values(): + plt.close(fig) + + # symex exported as CSV instead of plot + symex_csv_path = os.path.join(plots_dir, "symex_results.csv") + plotter.export_symex_csv(symex_csv_path) + + self.db.close() + + zip_base = os.path.join(self.artifacts_folder, self.run_name) + temp_name = os.path.join(self.artifacts_folder, "test_results") + os.rename(self.run_folder, temp_name) + shutil.make_archive( + zip_base, "zip", self.artifacts_folder, "test_results" + ) + shutil.rmtree(temp_name) + + print(f"\nArtifacts saved to: {zip_base}.zip") + + def setup(self): + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + run_num = self._get_next_run_number() + self.run_name = f"{run_num}-{timestamp}" + self.run_folder_name = f"test_results-{self.run_name}" + self.run_folder = os.path.join(self.artifacts_folder, self.run_folder_name) + self.run_tests_folder = os.path.join(self.run_folder, "tests") + self.runner.compiled_folder = os.path.join( + self.run_folder, "compiled_test_files" + ) + db_path = os.path.join(self.run_folder, "results.db") + + os.makedirs(self.run_folder, exist_ok=True) + os.makedirs(self.run_tests_folder, exist_ok=True) + os.makedirs(self.runner.compiled_folder, exist_ok=True) + + for f in os.listdir(self.tests_folder): + if f.endswith(".trv") or f.endswith(".trv.output") or f.endswith(".wh"): + shutil.copy(os.path.join(self.tests_folder, f), self.run_tests_folder) + + self.db = ResultsDatabase(db_path) + self.run_id = self.db.create_run(run_num, self.tests_folder) + + return self.run_folder + + def _get_next_run_number(self): + if not os.path.exists(self.artifacts_folder): + return 1 + + max_num = 0 + for item in os.listdir(self.artifacts_folder): + base_name = item[:-4] if item.endswith(".zip") else item + if os.path.isdir( + os.path.join(self.artifacts_folder, item) + ) or item.endswith(".zip"): + parts = base_name.split("-", 1) + if len(parts) == 2 and parts[0].isdigit(): + max_num = max(max_num, int(parts[0])) + return max_num + 1 + + +def run_setup(target="all", force=False): + script_dir = os.path.dirname(os.path.abspath(__file__)) + setup_module = os.path.join(script_dir, "src", "setup.py") + + cmd = [sys.executable, setup_module, "--target", target] + if force: + cmd.append("-f") + + result = subprocess.run(cmd) + return result.returncode + + +def get_missing_binaries(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + bin_dir = os.path.join(script_dir, "bin") + required_bins = ["minimc", "compiler", "interpreter"] + missing = [] + for bin_name in required_bins: + if not os.path.isfile(os.path.join(bin_dir, bin_name)): + missing.append(bin_name) + return missing + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=str, default=None, help="Path to config.toml") + parser.add_argument( + "--setup", + type=str, + nargs="?", + const="all", + default=None, + metavar="TARGETS", + help="Only run setup (build dependencies), then quit. Comma-separated: compiler, interpreter, minimc, or all", + ) + parser.add_argument( + "--skip-run", + action="store_true", + help="Only setup and compile, skip running tests", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Limit number of injection points to test", + ) + parser.add_argument( + "--nohard", + action="store_true", + help="Skip hardened version when running tests", + ) + parser.add_argument("--hard", action="store_true", help="Only run hardened version") + args, unknown = parser.parse_known_args() + + force = "-f" in unknown or "--reinit" in unknown + + if args.setup: + exit(run_setup(target=args.setup, force=force)) + + missing_bins = get_missing_binaries() + if missing_bins: + print(f"\033[91mError: Binaries not found: {', '.join(missing_bins)}\033[0m") + if len(missing_bins) > 1: + print("Run 'test --setup' to compile the binaries") + exit(1) + print("Run 'test --setup' to compile the binary") + exit(1) + + + if args.nohard and args.hard: + print("Error: Cannot use both --nohard and --hard") + exit() + + if args.hard: + run_variants = "hard" + elif args.nohard: + run_variants = "normal" + else: + run_variants = "both" + exp = Experiment(args, run_variants) + exp.run() diff --git a/testing/src/database.py b/testing/src/database.py index 8eadb3c..891452d 100644 --- a/testing/src/database.py +++ b/testing/src/database.py @@ -36,6 +36,7 @@ class ResultsDatabase: expected_output TEXT NOT NULL, matched_output_line TEXT, passed INTEGER NOT NULL, + edited INTEGER NOT NULL DEFAULT 0, FOREIGN KEY (run_id) REFERENCES runs(id) ); @@ -61,6 +62,7 @@ class ResultsDatabase: expected_output TEXT NOT NULL, matched_output_line TEXT, passed INTEGER NOT NULL, + edited INTEGER NOT NULL DEFAULT 0, FOREIGN KEY (run_id) REFERENCES runs(id) ); @@ -73,22 +75,26 @@ class ResultsDatabase: stmt INTEGER NOT NULL, faulty_bit INTEGER, result INTEGER NOT NULL, + edited INTEGER NOT NULL DEFAULT 0, FOREIGN KEY (run_id) REFERENCES runs(id) ); CREATE INDEX IF NOT EXISTS idx_test_results_run_id ON exhaustive_interpreter_results(run_id); CREATE INDEX IF NOT EXISTS idx_test_results_test ON exhaustive_interpreter_results(test); CREATE INDEX IF NOT EXISTS idx_test_results_variant ON exhaustive_interpreter_results(variant); CREATE INDEX IF NOT EXISTS idx_test_results_passed ON exhaustive_interpreter_results(passed); + CREATE INDEX IF NOT EXISTS idx_test_results_edited ON exhaustive_interpreter_results(edited); CREATE INDEX IF NOT EXISTS idx_test_results_run_id ON guided_interpreter_results(run_id); CREATE INDEX IF NOT EXISTS idx_test_results_test ON guided_interpreter_results(test); CREATE INDEX IF NOT EXISTS idx_test_results_variant ON guided_interpreter_results(variant); CREATE INDEX IF NOT EXISTS idx_test_results_passed ON guided_interpreter_results(passed); + CREATE INDEX IF NOT EXISTS idx_test_results_edited ON guided_interpreter_results(edited); CREATE INDEX IF NOT EXISTS idx_test_results_run_id ON symex_results(run_id); CREATE INDEX IF NOT EXISTS idx_test_results_test ON symex_results(test); CREATE INDEX IF NOT EXISTS idx_test_results_variant ON symex_results(variant); CREATE INDEX IF NOT EXISTS idx_test_results_result ON symex_results(result); + CREATE INDEX IF NOT EXISTS idx_test_results_edited ON symex_results(edited); """ def __init__(self, db_path): @@ -99,6 +105,22 @@ def __init__(self, db_path): def _init_db(self): self._conn = sqlite3.connect(self.db_path) self._conn.executescript(self.SCHEMA) + + cursor = self._conn.execute("PRAGMA table_info(symex_results)") + columns = [row[1] for row in cursor.fetchall()] + if "edited" not in columns: + self._conn.execute("ALTER TABLE symex_results ADD COLUMN edited INTEGER NOT NULL DEFAULT 0") + + cursor = self._conn.execute("PRAGMA table_info(exhaustive_interpreter_results)") + columns = [row[1] for row in cursor.fetchall()] + if "edited" not in columns: + self._conn.execute("ALTER TABLE exhaustive_interpreter_results ADD COLUMN edited INTEGER NOT NULL DEFAULT 0") + + cursor = self._conn.execute("PRAGMA table_info(guided_interpreter_results)") + columns = [row[1] for row in cursor.fetchall()] + if "edited" not in columns: + self._conn.execute("ALTER TABLE guided_interpreter_results ADD COLUMN edited INTEGER NOT NULL DEFAULT 0") + self._conn.commit() def close(self): @@ -120,8 +142,9 @@ def create_run(self, run_number, tests_folder=None): ) self._conn.commit() return cursor.lastrowid - def insert_symex_results_batch(self, run_id, results): + def insert_symex_results_batch(self, run_id, results, edited=False): rows = [] + edited_flag = 1 if edited else 0 for res in results: rows.append(( @@ -131,13 +154,14 @@ def insert_symex_results_batch(self, run_id, results): res["stmt"], res["faulty_bit"], res["result"], + edited_flag, )) self._conn.executemany( """ INSERT INTO symex_results - (run_id, test, variant, stmt, faulty_bit, result) - VALUES (?, ?, ?, ?, ?, ?) + (run_id, test, variant, stmt, faulty_bit, result, edited) + VALUES (?, ?, ?, ?, ?, ?, ?) """, rows, ) @@ -146,8 +170,9 @@ def insert_symex_results_batch(self, run_id, results): def query_df(self, query): return pd.read_sql_query(query, self._conn) - def insert_interpreter_results_batch(self, run_id, results, guided=False): + def insert_interpreter_results_batch(self, run_id, results, guided=False, edited=False): rows = [] + edited_flag = 1 if edited else 0 for res in results: parts = res["injection_point"].split(":") if parts[0] == "pc": @@ -203,6 +228,7 @@ def _clamp_int(val, max_val=2**63 - 1, min_val=-(2**63)): res["expected_output"], res.get("matched_output_line"), int(res["passed"]), + edited_flag, )) table = "exhaustive_interpreter_results" @@ -212,8 +238,8 @@ def _clamp_int(val, max_val=2**63 - 1, min_val=-(2**63)): f""" INSERT INTO {table} (run_id, test, variant, injection_point, fault_type, fault_pc, fault_reg, fault_bit, - returncode, pc_before, pc_after, expected_exec_instr, actual_exec_instr, reg_old_value, reg_new_value, stdout, stderr, expected_output, matched_output_line, passed) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + returncode, pc_before, pc_after, expected_exec_instr, actual_exec_instr, reg_old_value, reg_new_value, stdout, stderr, expected_output, matched_output_line, passed, edited) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, rows, ) diff --git a/testing/src/plotenator.py b/testing/src/plotenator.py index f2348c3..14499bd 100644 --- a/testing/src/plotenator.py +++ b/testing/src/plotenator.py @@ -1,97 +1,293 @@ +from matplotlib.axes import Axes from src.database import ResultsDatabase import matplotlib.pyplot as plt +import pandas as pd +import textwrap + +OUTCOME_COLORS = ["#ff2a2c", "#377eb8", "#4daf4a", "#984ea3", "#ff7f00", "#C9A227", "#a65628"] +FAULT_TYPE_COLORS = ["#a6cee3", "#ff4411", "#b2df8a"] class Plotter: def __init__(self, db: ResultsDatabase): self.db = db def create_plots(self): - print("Running plots:") return { - "exhaustive": self.plot_exhaustive(), - "guided": self.plot_guided() + "exhaustive": self.plot_exhaustive(edited=False), + "exhaustive_edited": self.plot_exhaustive(edited=True), + "guided": self.plot_guided(edited=False), + "guided_edited": self.plot_guided(edited=True), + "exhaustive_fault_types_normal": self.plot_fault_types(edited=False, table="exhaustive", variant="normal"), + "exhaustive_fault_types_hard": self.plot_fault_types(edited=False, table="exhaustive", variant="hard"), + "exhaustive_fault_types_normal_edited": self.plot_fault_types(edited=True, table="exhaustive", variant="normal"), + "guided_fault_types_normal": self.plot_fault_types(edited=False, table="guided", variant="normal"), + "guided_fault_types_hard": self.plot_fault_types(edited=False, table="guided", variant="hard"), + "guided_fault_types_normal_edited": self.plot_fault_types(edited=True, table="guided", variant="normal"), } - def plot_exhaustive(self): - df = self.db.query_df("SELECT * FROM exhaustive_interpreter_results") + def plot_exhaustive(self, edited=False): + suffix = " (Guided Manual Hardening)" if edited else "" + where = "WHERE edited = 1" if edited else "WHERE edited = 0" + df = self.db.query_df(f"SELECT * FROM exhaustive_interpreter_results {where}") - # Group by variant and passed - grouped = df.groupby(["variant", "passed"]).size().unstack(fill_value=0) + if df.empty: + fig, ax = plt.subplots() + ax.text(0.5, 0.5, f"No data{suffix}", ha='center', va='center', transform=ax.transAxes) + ax.set_title(f"Exhaustive Interpreter - Outcome Distribution{suffix}") + return fig - # Map the numeric passed values to descriptive labels - passed_labels = { - 0: "Faulty", - 1: "Normal Execution", - 2: "Panicked!", + known_codes = {0, 1, 2, 3, 77, 78} + df = df.copy() + df["outcome"] = df["returncode"].apply( + lambda rc: rc if rc in known_codes else "Other" + ) + + returncode_labels = { + 0: "Normal Execution", + 1: "Faulty", + 2: "Panicked", + 3: "Infinite Loop", 77: "Countermeasure Activated", - 78: "Faulty Jump to Countermeasure" + 78: "Faulty Jump to Countermeasure", + "Other": "Other", } - # Rename the columns - grouped = grouped.rename(columns=passed_labels) + grouped = df.groupby(["variant", "outcome"]).size().unstack(fill_value=0) + grouped = grouped.rename(columns=returncode_labels) + + col_order = [c for c in returncode_labels.values() if c != "Other" and c in grouped.columns] + if "Other" in grouped.columns: + col_order.append("Other") + grouped = grouped[col_order] - # Plot fig, ax = plt.subplots() - grouped.plot(kind="bar", stacked=False, ax=ax) + grouped.plot(kind="bar", stacked=False, ax=ax, color=OUTCOME_COLORS[:len(grouped.columns)], legend=False) - ax.set_title("Exhaustive Interpreter - Outcome Distribution") - ax.set_ylabel("Count (log scale)") - ax.set_xlabel("Variant") - ax.set_yscale("log") - ax.legend(title="Outcome") + for container in ax.containers: + ax.bar_label( + container, + labels=[ + f'{int(v.get_height())}' if v.get_height() > 0 else '' + for v in container + ], + label_type='center', + fontsize=8, + rotation=90, + color='black', + fontweight='bold' + ) + + ax.set_title(f"Exhaustive Interpreter - Outcome Distribution{suffix}") + ax.set_xlabel(None) + self._apply_scale(ax) + self._add_horizontal_legend(fig, ax, "Outcome") self._style_axes(ax) return fig - def plot_guided(self): - df = self.db.query_df("SELECT * FROM guided_interpreter_results") + def plot_guided(self, edited=False): + suffix = " (Guided Manual Hardening)" if edited else "" + where = "WHERE edited = 1" if edited else "WHERE edited = 0" + df = self.db.query_df(f"SELECT * FROM guided_interpreter_results {where}") - grouped = df.groupby(["variant", "passed"]).size().unstack(fill_value=0) + if df.empty: + fig, ax = plt.subplots() + ax.text(0.5, 0.5, f"No data{suffix}", ha='center', va='center', transform=ax.transAxes) + ax.set_title(f"Guided Interpreter - Outcome Distribution{suffix}") + return fig - # Map the numeric passed values to descriptive labels - passed_labels = { - 0: "Faulty", - 1: "Normal Execution", - 2: "Panicked!", + known_codes = {0, 1, 2, 3, 77, 78} + df = df.copy() + df["outcome"] = df["returncode"].apply( + lambda rc: rc if rc in known_codes else "Other" + ) + + returncode_labels = { + 0: "Normal Execution", + 1: "Faulty", + 2: "Panicked", + 3: "Infinite Loop", 77: "Countermeasure Activated", - 78: "Faulty Jump to Countermeasure" + 78: "Faulty Jump to Countermeasure", + "Other": "Other", } - grouped = grouped.rename(columns=passed_labels) + grouped = df.groupby(["variant", "outcome"]).size().unstack(fill_value=0) + grouped = grouped.rename(columns=returncode_labels) + + col_order = [c for c in returncode_labels.values() if c != "Other" and c in grouped.columns] + if "Other" in grouped.columns: + col_order.append("Other") + grouped = grouped[col_order] fig, ax = plt.subplots() - grouped.plot(kind="bar", stacked=False, ax=ax) + grouped.plot(kind="bar", stacked=False, ax=ax, color=OUTCOME_COLORS[:len(grouped.columns)], legend=False) - ax.set_title("Guided Interpreter - Outcome Distribution") - ax.set_ylabel("Count (log scale)") - ax.set_xlabel("Variant") - ax.set_yscale("log") - ax.legend(title="Passed value") + for container in ax.containers: + ax.bar_label( + container, + labels=[ + f'{int(v.get_height())}' if v.get_height() > 0 else '' + for v in container + ], + label_type='center', + fontsize=8, + rotation=90, + color='black', + fontweight='bold' + ) + + ax.set_title(f"Guided Interpreter - Outcome Distribution{suffix}") + ax.set_xlabel(None) + self._apply_scale(ax) + self._add_horizontal_legend(fig, ax, "Outcome") self._style_axes(ax) return fig - def _style_axes(self, ax): - # axis labels + def _style_axes(self, ax: Axes): ax.xaxis.label.set_fontweight('bold') ax.yaxis.label.set_fontweight('bold') + ax.tick_params(axis='x', rotation=0) + + # 1. Extract, wrap, and update the text of each label + wrapped_labels = [] + for label in ax.get_xticklabels(): + text = label.get_text() + # Wrap at 12–15 characters depending on your font size + wrapped_text = textwrap.fill(text, width=14) + wrapped_labels.append(wrapped_text) + + # 2. Re-apply the wrapped strings back to the axis + ax.set_xticklabels(wrapped_labels, ha='center') - # # tick labels (numbers, "hard", "normal", etc.) - # for label in ax.get_xticklabels(): - # label.set_fontweight('bold') + def _apply_scale(self, ax: Axes, ratio_threshold=10): + heights = [] + for container in ax.containers: + for patch in container: + h = patch.get_height() + if h > 0: + heights.append(h) + if not heights: + ax.set_ylabel("Count") + return + ratio = max(heights) / min(heights) + if ratio > ratio_threshold: + ax.set_yscale("log") + ax.set_ylabel("Count (log scale)") + ymin, ymax = ax.get_ylim() + else: + ax.set_ylabel("Count") + ymin, ymax = ax.get_ylim() - # for label in ax.get_yticklabels(): - # label.set_fontweight('bold') + def _add_horizontal_legend(self, fig, ax, title, wrap_width=14): + handles, labels = ax.get_legend_handles_labels() + wrapped = [textwrap.fill(l, width=wrap_width) for l in labels] + leg = fig.legend( + handles, wrapped, + title=title, + loc='lower center', + bbox_to_anchor=(0.5, 0.05), + ncol=min(len(handles), 4), + fontsize=8, + title_fontsize=9, + frameon=True, + ) + leg.get_title().set_fontweight('bold') + fig.tight_layout() + fig.subplots_adjust(bottom=0.25) - # legend - # legend = ax.get_legend() - # if legend: - # for text in legend.get_texts(): - # text.set_fontweight('bold') def export_symex_csv(self, output_path): df = self.db.query_df("SELECT * FROM symex_results") + column_mapping = { + 'run_id': 'ID', + 'variant': 'Variant', + 'stmt': 'Stmt', + 'faulty_bit': 'Faulty Bit', + 'result': 'Result', + } + df = df.rename(columns=column_mapping) + df.loc[df['edited'] == 1, 'Variant'] = 'Manual Hard' + desired_columns = ['Variant', 'Stmt', 'Faulty Bit', 'Result'] + df = df[[col for col in desired_columns if col in df.columns]] df.to_csv(output_path, index=False) + + def plot_fault_types(self, edited=False, table="exhaustive", variant="normal"): + suffix = " (Guided Manual Hardening)" if edited else "" + variant_label = variant + where = f"WHERE edited = {1 if edited else 0} AND variant = '{variant}'" + table_name = f"{table}_interpreter_results" + df = self.db.query_df(f"SELECT * FROM {table_name} {where}") + + if df.empty: + fig, ax = plt.subplots() + display_label = variant.title() + ax.text(0.5, 0.5, f"No data ({display_label}){suffix}", ha='center', va='center', transform=ax.transAxes) + ax.set_title(f"Fault Type Distribution - {display_label}{suffix}") + return fig + + fault_outcomes = df.copy() + + known_codes = {0, 1, 2, 3, 77, 78} + fault_outcomes["outcome"] = fault_outcomes["returncode"].apply( + lambda rc: rc if rc in known_codes else "Other" + ) + + returncode_labels = { + 0: "Normal Execution", + 1: "Faulty", + 2: "Panicked!", + 3: "Infinite Loop", + 77: "Countermeasure Activated", + 78: "Faulty Jump to Countermeasure", + "Other": "Other", + } + fault_outcomes["outcome"] = fault_outcomes["outcome"].map(returncode_labels) + + fault_type_map = {"pc": "PC", "reg": "REG", "cpsr": "CPSR"} + fault_outcomes["fault_type_label"] = fault_outcomes["fault_type"].map(fault_type_map) + + grouped = fault_outcomes.groupby(["outcome", "fault_type_label"]).size().unstack(fill_value=0) + + ordered_outcomes = [c for c in returncode_labels.values() if c != "Other" and c in grouped.index] + if "Other" in grouped.index: + ordered_outcomes.append("Other") + grouped = grouped.reindex(ordered_outcomes) + + ordered_cols = ["PC", "REG", "CPSR"] + for col in ordered_cols: + if col not in grouped.columns: + grouped[col] = 0 + grouped = grouped[ordered_cols] + + fig, ax = plt.subplots() + grouped.plot(kind="bar", stacked=False, ax=ax, color=FAULT_TYPE_COLORS[:len(grouped.columns)], legend=False) + + for container in ax.containers: + ax.bar_label( + container, + labels=[ + f'{int(v.get_height())}' if v.get_height() > 0 else '' + for v in container + ], + label_type='center', + fontsize=8, + rotation=90, + color='black', + fontweight='bold' + ) + + interpreter_name = "Exhaustive" if table == "exhaustive" else "Guided" + display_label = variant.title() + ax.set_title(f"{interpreter_name} - Fault Type Distribution - {display_label}{suffix}") + ax.set_xlabel(None) + self._apply_scale(ax) + self._add_horizontal_legend(fig, ax, "Fault Type") + + self._style_axes(ax) + + return fig diff --git a/testing/src/testrunner.py b/testing/src/testrunner.py index 965206f..8503a9e 100644 --- a/testing/src/testrunner.py +++ b/testing/src/testrunner.py @@ -106,13 +106,13 @@ def _check_passed(result, pass_mode_config, cm_start, cm_end, injection_point): expected_pass = pass_mode_config.get("expected_pass", None) expected_fail = pass_mode_config.get("expected_fail", None) - if "COUNTERMEASURE" in result.stdout: + if "COUNTERMEASURE" in result.stdout or result.returncode == 77: return 77 elif "Detected Infinite Loop" in result.stdout: return 3 if mode == "returncode" and result.returncode == 0: return 1 - elif mode == "returncode" and "panic" in result.stderr: + elif "panic" in result.stderr: return 2 elif mode == "returncode" and result.returncode != 0: if cm_start is not None and cm_end is not None and injection_point is not None: @@ -175,7 +175,6 @@ def __init__(self, config_path=None): self.pass_mode_configs = config.get("pass_mode", {}) self.tests = self._discover_tests() self.compile_results = None - self.current_file : File = None def _discover_tests(self): tests = [] @@ -352,8 +351,14 @@ def _generate_injection_points(self, asm_path): registers = [] cpsr_registers = [] pc_points = [] + do_all = False + + if len(self.injection_points) == 0: + do_all = True for point in self.injection_points: + if do_all: + break if point.startswith('cpsr_'): cpsr_registers.append(point[5:]) elif point.startswith('r') and point[1:].isdigit(): @@ -367,8 +372,8 @@ def _generate_injection_points(self, asm_path): pc_points.append(point[3:]) - if len(pc_points) > 0: - if pc_points[0] == "ALL": + if len(pc_points) > 0 or do_all: + if do_all or pc_points[0] == "ALL": for pc in range(1, max_pc): for bit in range(0, 32): injection_points.append(f"pc:{pc}:{bit}") @@ -377,21 +382,33 @@ def _generate_injection_points(self, asm_path): for bit in range(0, 32): injection_points.append(f"pc:{pc}:{bit}") - if len(registers) > 0: - for pc in range(1, max_pc): - for reg in registers: - for bit in range(0, 32): - injection_points.append(f"reg:{pc}:{reg}:{bit}") + if len(registers) > 0 or do_all: + if do_all: + for pc in range(1, max_pc): + for reg in range(0, 13): + for bit in range(0, 32): + injection_points.append(f"reg:{pc}:{reg}:{bit}") + else: + for pc in range(1, max_pc): + for reg in registers: + for bit in range(0, 32): + injection_points.append(f"reg:{pc}:{reg}:{bit}") - if len(cpsr_registers) > 0: - for pc in range(1, max_pc): - for reg in cpsr_registers: - injection_points.append(f"cpsr:{pc}:{reg}") + if len(cpsr_registers) > 0 or do_all: + if do_all: + for pc in range(1, max_pc): + for reg in ["n","z","c", "v"]: + injection_points.append(f"cpsr:{pc}:{reg}") + else: + for pc in range(1, max_pc): + for reg in cpsr_registers: + injection_points.append(f"cpsr:{pc}:{reg}") return injection_points def stmt_to_pc_range(self, asm_path, stmt: int): with open(asm_path, 'r') as f: + print(f"Looking for STMT={stmt} in file: {asm_path}") lines = f.readlines() for i, line in enumerate(lines): if f'.word 0x10000000 @ {stmt}' in line: @@ -422,12 +439,6 @@ def _symex_to_injection_points(self, variant, asm_path, minimc_res): for pc in range(start_pc, end_pc+1): ip = f"reg:{pc}:{reg}:{bit}" injection_points.append(ip) - for pc in range(start_pc, end_pc+1): - for bit in range(0, 32): - ip = f"pc:{pc}:{bit}" - injection_points.append(ip) - for cpsr in ["n","v","z","c"]: - ip = f"cpsr:{pc}:{cpsr}" return injection_points @@ -468,7 +479,7 @@ def _run_fault_campaign(self, test_name, variant, asm_path, pass_mode_config, li return results - def compile(self): + def compile(self, nohard=False): results = [] for test in self.tests: test_result = {"name": test["name"], "normal": None, "hard": None} diff --git a/testing/test b/testing/test deleted file mode 100755 index 2bd702d..0000000 --- a/testing/test +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/python3 - -import argparse, os, toml, shutil, subprocess, sys -import matplotlib.pyplot as plt -from datetime import datetime -from src.database import ResultsDatabase -from src.testrunner import TestRunner -from src.symexrunner import SymexRunner -from src.plotenator import Plotter -from rich.console import Console - - -class Experiment: - def __init__(self, args, run_variants, config_path=None): - if config_path is None: - config_path = os.path.join(os.path.dirname(__file__), "config.toml") - - with open(config_path, "r") as f: - config = toml.load(f) - self.args = args - self.run_variants = run_variants - self.base_dir = os.path.dirname(__file__) - self.tests_folder = os.path.join( - self.base_dir, config.get("test_folder", "fissc") - ) - self.artifacts_folder = os.path.join(self.base_dir, "artifacts") - self.plot_mode = config.get("plotmode", "save") - self.injection_points = config.get( - "injection_points", - [ - "r0", - "r1", - "r2", - "r3", - "r4", - "r5", - "r6", - "r7", - "r8", - "r9", - "r10", - "r11", - "r12", - ], - ) - self.default_pass_mode = config.get("pass_mode", {}).get( - "default", "returncode" - ) - self.pass_mode_configs = config.get("pass_mode", {}) - self.runner = TestRunner(config_path=args.config) - self.tests = self.runner._discover_tests() - self.compile_results = None - - def run(self): - - # check that the compiler and interpreter is present: - self.runner.check_bin() - - print(f"Setting up...") - folder = self.setup() - print(f"Created folder: {folder}") - - print(f"\nCompiling tests...") - results = self.runner.compile() - print(f"Compiled {len(results)} tests") - all_fault_results = None - wh_files = sorted( - [f for f in os.listdir(self.runner.compiled_folder) if f.endswith(".wh")] - ) - - print("Whiley Files are ready for adding fault injections...") - print("\nRunning symex...") - symex_runner = SymexRunner(config_path=self.args.config) - symex_runner.artifacts_folder = self.runner.compiled_folder - symex_results = symex_runner.run(self.run_variants) - - self.db.insert_symex_results_batch(self.run_id, symex_results) - - if not self.args.skip_run: - print(f"\nRunning interpreter ...") - exhaustive_fault_results = self.runner.run_tests( - limit=self.args.limit, run_variants=run_variants - ) - guided_fault_results = self.runner.run_tests(limit=self.args.limit, run_variants=run_variants, minimc_res=symex_results) - else: - print(f"\nSkipping test run (--skip-run)") - - self.db.insert_interpreter_results_batch(self.run_id, exhaustive_fault_results) - self.db.insert_interpreter_results_batch(self.run_id, guided_fault_results, True) - - plotter = Plotter(self.db) - plots = plotter.create_plots() - - - plots_dir = os.path.join(self.run_folder, "plots") - os.makedirs(plots_dir, exist_ok=True) - - - def save_plot(name, fig): - """Handles the saving of the figure to a specific directory.""" - path = os.path.join(plots_dir, f"{name}.svg") - fig.savefig(path, format="svg", bbox_inches="tight") - - # only these remain as plots - if self.plot_mode == "save" or self.plot_mode == "both": - save_plot("exhaustive", plots["exhaustive"]) - save_plot("guided", plots["guided"]) - - if self.plot_mode == "show" or self.plot_mode == "both": - plt.show() - for fig in plots.values(): - plt.close(fig) - - # symex exported as CSV instead of plot - symex_csv_path = os.path.join(plots_dir, "symex_results.csv") - plotter.export_symex_csv(symex_csv_path) - - self.db.close() - - zip_base = os.path.join(self.artifacts_folder, self.run_name) - shutil.make_archive( - zip_base, "zip", self.artifacts_folder, self.run_folder_name - ) - shutil.rmtree(self.run_folder) - - print(f"\nArtifacts saved to: {zip_base}.zip") - - def setup(self): - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - run_num = self._get_next_run_number() - self.run_name = f"{run_num}-{timestamp}" - self.run_folder_name = f"test_results" - self.run_folder = os.path.join(self.artifacts_folder, self.run_folder_name) - self.run_tests_folder = os.path.join(self.run_folder, "tests") - self.runner.compiled_folder = os.path.join( - self.run_folder, "compiled_test_files" - ) - db_path = os.path.join(self.run_folder, "results.db") - - os.makedirs(self.run_folder, exist_ok=True) - os.makedirs(self.run_tests_folder, exist_ok=True) - os.makedirs(self.runner.compiled_folder, exist_ok=True) - - for f in os.listdir(self.tests_folder): - if f.endswith(".trv") or f.endswith(".trv.output") or f.endswith(".wh"): - shutil.copy(os.path.join(self.tests_folder, f), self.run_tests_folder) - - self.db = ResultsDatabase(db_path) - self.run_id = self.db.create_run(run_num, self.tests_folder) - - return self.run_folder - - def _get_next_run_number(self): - if not os.path.exists(self.artifacts_folder): - return 1 - - max_num = 0 - for item in os.listdir(self.artifacts_folder): - base_name = item[:-4] if item.endswith(".zip") else item - if os.path.isdir( - os.path.join(self.artifacts_folder, item) - ) or item.endswith(".zip"): - parts = base_name.split("-", 1) - if len(parts) == 2 and parts[0].isdigit(): - max_num = max(max_num, int(parts[0])) - return max_num + 1 - - -def run_setup(target="all", force=False): - script_dir = os.path.dirname(os.path.abspath(__file__)) - setup_module = os.path.join(script_dir, "src", "setup.py") - - cmd = [sys.executable, setup_module, "--target", target] - if force: - cmd.append("-f") - - result = subprocess.run(cmd) - return result.returncode - - -def get_missing_binaries(): - script_dir = os.path.dirname(os.path.abspath(__file__)) - bin_dir = os.path.join(script_dir, "bin") - required_bins = ["minimc", "compiler", "interpreter"] - missing = [] - for bin_name in required_bins: - if not os.path.isfile(os.path.join(bin_dir, bin_name)): - missing.append(bin_name) - return missing - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--config", type=str, default=None, help="Path to config.toml") - parser.add_argument( - "--setup", - type=str, - nargs="?", - const="all", - default=None, - metavar="TARGETS", - help="Only run setup (build dependencies), then quit. Comma-separated: compiler, interpreter, minimc, or all", - ) - parser.add_argument( - "--skip-run", - action="store_true", - help="Only setup and compile, skip running tests", - ) - parser.add_argument( - "--limit", - type=int, - default=None, - help="Limit number of injection points to test", - ) - parser.add_argument( - "--nohard", - action="store_true", - help="Skip hardened version when running tests", - ) - parser.add_argument("--hard", action="store_true", help="Only run hardened version") - args, unknown = parser.parse_known_args() - - force = "-f" in unknown or "--reinit" in unknown - - if args.setup: - exit(run_setup(target=args.setup, force=force)) - - missing_bins = get_missing_binaries() - if missing_bins: - print(f"\033[91mError: Binaries not found: {', '.join(missing_bins)}\033[0m") - if len(missing_bins) > 1: - print("Run 'test --setup' to compile the binaries") - exit(1) - print("Run 'test --setup' to compile the binary") - exit(1) - - - if args.nohard and args.hard: - print("Error: Cannot use both --nohard and --hard") - exit() - - if args.hard: - run_variants = "hard" - elif args.nohard: - run_variants = "normal" - else: - run_variants = "both" - exp = Experiment(args, run_variants) - exp.run() diff --git a/testing/test_results/plots/symex_results.csv b/testing/test_results/plots/symex_results.csv deleted file mode 100644 index 370411f..0000000 --- a/testing/test_results/plots/symex_results.csv +++ /dev/null @@ -1,17 +0,0 @@ -id,run_id,test,variant,stmt,faulty_bit,result -1,2,pinny,Hard,9,2,77 -2,2,pinny,Hard,8,2,77 -3,2,pinny,Hard,6,1,77 -4,2,pinny,Hard,4,2,77 -5,2,pinny,Hard,2,1,77 -6,2,pinny,Hard,0,0,1 -7,2,pinny,Normal,9,2,1 -8,2,pinny,Normal,0,0,1 -9,3,pinny,Hard,9,2,77 -10,3,pinny,Hard,8,2,77 -11,3,pinny,Hard,6,1,77 -12,3,pinny,Hard,4,2,77 -13,3,pinny,Hard,2,1,77 -14,3,pinny,Hard,0,0,1 -15,3,pinny,Normal,9,2,1 -16,3,pinny,Normal,0,0,1