From ac18c6c8c4e6672c318d05870562ee6b2fd1257c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 22 Jul 2026 20:39:30 +0400 Subject: [PATCH 1/5] feat(harness): track successful no-progress loops --- src/harness/mod.rs | 4 +- src/harness/no_progress/mod.rs | 5 + src/harness/no_progress/successful_repeat.rs | 222 +++++++++++++++++++ 3 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 src/harness/no_progress/successful_repeat.rs diff --git a/src/harness/mod.rs b/src/harness/mod.rs index 43fac9b..217eeea 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -52,7 +52,9 @@ pub use model::{ collect_model_stream, context_window_for_model_id, }; pub use no_progress::{ - DEFAULT_IDENTICAL_HALT_THRESHOLD, NoProgress, NoProgressTracker, ToolAttempt, + DEFAULT_IDENTICAL_HALT_THRESHOLD, DEFAULT_REPEAT_CALL_THRESHOLD, + DEFAULT_REPEAT_OUTPUT_THRESHOLD, NoProgress, NoProgressTracker, SuccessfulRepeat, + SuccessfulRepeatTracker, ToolAttempt, }; pub use tool::{ Tool as HarnessTool, ToolCall as HarnessToolCall, ToolFormat, ToolRegistry, diff --git a/src/harness/no_progress/mod.rs b/src/harness/no_progress/mod.rs index 981d2c8..4ef7dd9 100644 --- a/src/harness/no_progress/mod.rs +++ b/src/harness/no_progress/mod.rs @@ -19,8 +19,13 @@ //! and turns the returned [`NoProgress`] verdict into a steering nudge //! (`Nudge`) or a halt (`Halt`). +mod successful_repeat; mod types; +pub use successful_repeat::{ + DEFAULT_REPEAT_CALL_THRESHOLD, DEFAULT_REPEAT_OUTPUT_THRESHOLD, SuccessfulRepeat, + SuccessfulRepeatTracker, +}; use types::LadderState; pub use types::{NoProgress, NoProgressTracker, ToolAttempt}; diff --git a/src/harness/no_progress/successful_repeat.rs b/src/harness/no_progress/successful_repeat.rs new file mode 100644 index 0000000..697c6e8 --- /dev/null +++ b/src/harness/no_progress/successful_repeat.rs @@ -0,0 +1,222 @@ +//! Successful-repeat progress detection. +//! +//! [`NoProgressTracker`] handles failing tool calls, but deliberately resets on +//! success. That leaves a second loop shape undetected: a model can repeatedly +//! emit the same response and successfully invoke the same no-op tool call. +//! This tracker owns the provider- and product-neutral streak accounting for +//! those loops; a harness middleware remains responsible for building canonical +//! signatures and deciding which polling tools are exempt. + +use std::hash::{Hash, Hasher}; +use std::sync::Mutex; + +/// Identical assistant-output batches allowed before halting. +pub const DEFAULT_REPEAT_OUTPUT_THRESHOLD: u32 = 4; +/// Identical successful tool-call batches allowed before halting. +pub const DEFAULT_REPEAT_CALL_THRESHOLD: u32 = 3; + +/// Verdict returned after recording a successful-repeat signal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SuccessfulRepeat { + /// The signature changed, is exempt, failed, or remains below its threshold. + Continue, + /// The same successful action has repeated enough times to be considered + /// stuck. The message is suitable for steering or a halt summary. + Halt(String), +} + +#[derive(Default)] +struct Streak { + last_hash: Option, + consecutive: u32, +} + +impl Streak { + fn record(&mut self, signature: &str) -> u32 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + signature.hash(&mut hasher); + let hash = hasher.finish(); + if self.last_hash == Some(hash) { + self.consecutive += 1; + } else { + self.last_hash = Some(hash); + self.consecutive = 1; + } + self.consecutive + } + + fn reset(&mut self) { + *self = Self::default(); + } +} + +/// Tracks identical assistant-output and successful tool-call batches. +/// +/// The two streaks are independent: output is observed before tools execute, +/// while a call batch is recorded only after every result is known. Exempt +/// polling batches reset their streak; a failed call batch also resets the +/// successful-call streak so the failure ladder remains authoritative. +pub struct SuccessfulRepeatTracker { + output_threshold: u32, + call_threshold: u32, + output: Mutex, + calls: Mutex, +} + +impl Default for SuccessfulRepeatTracker { + fn default() -> Self { + Self::new( + DEFAULT_REPEAT_OUTPUT_THRESHOLD, + DEFAULT_REPEAT_CALL_THRESHOLD, + ) + } +} + +impl SuccessfulRepeatTracker { + /// Builds a tracker. Thresholds are clamped to one so `0` cannot disable a + /// safety guard accidentally; callers that do not want this guard should + /// omit the tracker. + pub fn new(output_threshold: u32, call_threshold: u32) -> Self { + Self { + output_threshold: output_threshold.max(1), + call_threshold: call_threshold.max(1), + output: Mutex::new(Streak::default()), + calls: Mutex::new(Streak::default()), + } + } + + /// Records the canonical visible-output plus tool-call signature produced + /// by one assistant iteration. + pub fn record_output(&self, signature: &str, exempt: bool) -> SuccessfulRepeat { + let mut output = self.output.lock().unwrap(); + if exempt { + output.reset(); + return SuccessfulRepeat::Continue; + } + let consecutive = output.record(signature); + if consecutive < self.output_threshold { + return SuccessfulRepeat::Continue; + } + SuccessfulRepeat::Halt(format!( + "Stopping: the last {consecutive} iterations produced the identical response and tool call with no change; the run is stuck repeating the same step without making progress." + )) + } + + /// Records the canonical tool-name/arguments signature after the whole + /// batch completes. Failed or exempt batches reset the successful streak. + pub fn record_call_batch( + &self, + signature: &str, + all_successful: bool, + exempt: bool, + ) -> SuccessfulRepeat { + let mut calls = self.calls.lock().unwrap(); + if exempt || !all_successful { + calls.reset(); + return SuccessfulRepeat::Continue; + } + let consecutive = calls.record(signature); + if consecutive < self.call_threshold { + return SuccessfulRepeat::Continue; + } + SuccessfulRepeat::Halt(format!( + "Stopping: the same successful tool-call batch was issued {consecutive} times in a row with identical arguments and no new information; the run is stuck repeating one action without making progress." + )) + } + + /// Clears both streaks, for example when a paused run is resumed. + pub fn reset(&self) { + self.output.lock().unwrap().reset(); + self.calls.lock().unwrap().reset(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn identical_output_halts_at_threshold_and_changes_reset() { + let tracker = SuccessfulRepeatTracker::new(3, 3); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert!(matches!( + tracker.record_output("same", false), + SuccessfulRepeat::Halt(message) if message.contains("3 iterations") + )); + assert_eq!( + tracker.record_output("different", false), + SuccessfulRepeat::Continue + ); + } + + #[test] + fn successful_call_batches_halt_but_failures_reset() { + let tracker = SuccessfulRepeatTracker::new(4, 2); + assert_eq!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Continue + ); + assert!(matches!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Halt(message) if message.contains("2 times") + )); + assert_eq!( + tracker.record_call_batch("tool:args", false, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Continue + ); + } + + #[test] + fn exempt_batches_reset_both_streaks() { + let tracker = SuccessfulRepeatTracker::new(2, 2); + assert_eq!( + tracker.record_output("poll", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("poll", true), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("poll", false), + SuccessfulRepeat::Continue + ); + + assert_eq!( + tracker.record_call_batch("poll", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, true), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, false), + SuccessfulRepeat::Continue + ); + } + + #[test] + fn zero_thresholds_are_fail_safe() { + let tracker = SuccessfulRepeatTracker::new(0, 0); + assert!(matches!( + tracker.record_output("same", false), + SuccessfulRepeat::Halt(_) + )); + assert!(matches!( + tracker.record_call_batch("same", true, false), + SuccessfulRepeat::Halt(_) + )); + } +} From b999021bd428f8daefcece8e781a313a7f56490d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 22 Jul 2026 20:44:15 +0400 Subject: [PATCH 2/5] fix(harness): reset output streak after failures --- src/harness/no_progress/successful_repeat.rs | 26 ++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/harness/no_progress/successful_repeat.rs b/src/harness/no_progress/successful_repeat.rs index 697c6e8..145102d 100644 --- a/src/harness/no_progress/successful_repeat.rs +++ b/src/harness/no_progress/successful_repeat.rs @@ -10,9 +10,9 @@ use std::hash::{Hash, Hasher}; use std::sync::Mutex; -/// Identical assistant-output batches allowed before halting. +/// Consecutive identical assistant-output batches required to halt. pub const DEFAULT_REPEAT_OUTPUT_THRESHOLD: u32 = 4; -/// Identical successful tool-call batches allowed before halting. +/// Consecutive identical successful tool-call batches required to halt. pub const DEFAULT_REPEAT_CALL_THRESHOLD: u32 = 3; /// Verdict returned after recording a successful-repeat signal. @@ -113,6 +113,10 @@ impl SuccessfulRepeatTracker { let mut calls = self.calls.lock().unwrap(); if exempt || !all_successful { calls.reset(); + drop(calls); + if !all_successful { + self.output.lock().unwrap().reset(); + } return SuccessfulRepeat::Continue; } let consecutive = calls.record(signature); @@ -177,6 +181,24 @@ mod tests { ); } + #[test] + fn failed_call_batches_reset_output_repeats() { + let tracker = SuccessfulRepeatTracker::new(2, 2); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("same-call", false, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "a failed prior batch must not count toward a successful output loop" + ); + } + #[test] fn exempt_batches_reset_both_streaks() { let tracker = SuccessfulRepeatTracker::new(2, 2); From 465f28a9d557581fe9ed2ae6d5f345c2b03e7397 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 22 Jul 2026 21:06:32 +0400 Subject: [PATCH 3/5] refactor(harness): centralize no-progress types --- src/harness/no_progress/mod.rs | 9 +++-- src/harness/no_progress/successful_repeat.rs | 36 +++----------------- src/harness/no_progress/types.rs | 29 ++++++++++++++++ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/harness/no_progress/mod.rs b/src/harness/no_progress/mod.rs index 4ef7dd9..8e98340 100644 --- a/src/harness/no_progress/mod.rs +++ b/src/harness/no_progress/mod.rs @@ -22,12 +22,11 @@ mod successful_repeat; mod types; -pub use successful_repeat::{ - DEFAULT_REPEAT_CALL_THRESHOLD, DEFAULT_REPEAT_OUTPUT_THRESHOLD, SuccessfulRepeat, - SuccessfulRepeatTracker, -}; +pub use successful_repeat::{DEFAULT_REPEAT_CALL_THRESHOLD, DEFAULT_REPEAT_OUTPUT_THRESHOLD}; use types::LadderState; -pub use types::{NoProgress, NoProgressTracker, ToolAttempt}; +pub use types::{ + NoProgress, NoProgressTracker, SuccessfulRepeat, SuccessfulRepeatTracker, ToolAttempt, +}; use std::sync::Mutex; diff --git a/src/harness/no_progress/successful_repeat.rs b/src/harness/no_progress/successful_repeat.rs index 145102d..f1ca6ce 100644 --- a/src/harness/no_progress/successful_repeat.rs +++ b/src/harness/no_progress/successful_repeat.rs @@ -8,29 +8,14 @@ //! signatures and deciding which polling tools are exempt. use std::hash::{Hash, Hasher}; -use std::sync::Mutex; + +use super::types::{Streak, SuccessfulRepeat, SuccessfulRepeatTracker}; /// Consecutive identical assistant-output batches required to halt. pub const DEFAULT_REPEAT_OUTPUT_THRESHOLD: u32 = 4; /// Consecutive identical successful tool-call batches required to halt. pub const DEFAULT_REPEAT_CALL_THRESHOLD: u32 = 3; -/// Verdict returned after recording a successful-repeat signal. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SuccessfulRepeat { - /// The signature changed, is exempt, failed, or remains below its threshold. - Continue, - /// The same successful action has repeated enough times to be considered - /// stuck. The message is suitable for steering or a halt summary. - Halt(String), -} - -#[derive(Default)] -struct Streak { - last_hash: Option, - consecutive: u32, -} - impl Streak { fn record(&mut self, signature: &str) -> u32 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); @@ -50,19 +35,6 @@ impl Streak { } } -/// Tracks identical assistant-output and successful tool-call batches. -/// -/// The two streaks are independent: output is observed before tools execute, -/// while a call batch is recorded only after every result is known. Exempt -/// polling batches reset their streak; a failed call batch also resets the -/// successful-call streak so the failure ladder remains authoritative. -pub struct SuccessfulRepeatTracker { - output_threshold: u32, - call_threshold: u32, - output: Mutex, - calls: Mutex, -} - impl Default for SuccessfulRepeatTracker { fn default() -> Self { Self::new( @@ -80,8 +52,8 @@ impl SuccessfulRepeatTracker { Self { output_threshold: output_threshold.max(1), call_threshold: call_threshold.max(1), - output: Mutex::new(Streak::default()), - calls: Mutex::new(Streak::default()), + output: std::sync::Mutex::new(Streak::default()), + calls: std::sync::Mutex::new(Streak::default()), } } diff --git a/src/harness/no_progress/types.rs b/src/harness/no_progress/types.rs index 94c8e73..b31cd52 100644 --- a/src/harness/no_progress/types.rs +++ b/src/harness/no_progress/types.rs @@ -63,3 +63,32 @@ pub struct NoProgressTracker { pub(super) identical_halt_threshold: usize, pub(super) state: Mutex, } + +/// Verdict returned after recording a successful-repeat signal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SuccessfulRepeat { + /// The signature changed, is exempt, failed, or remains below its threshold. + Continue, + /// The same successful action has repeated enough times to be considered + /// stuck. The message is suitable for steering or a halt summary. + Halt(String), +} + +#[derive(Default)] +pub(super) struct Streak { + pub(super) last_hash: Option, + pub(super) consecutive: u32, +} + +/// Tracks identical assistant-output and successful tool-call batches. +/// +/// The two streaks are independent: output is observed before tools execute, +/// while a call batch is recorded only after every result is known. Exempt +/// polling batches reset their streak; a failed call batch also resets the +/// successful-call streak so the failure ladder remains authoritative. +pub struct SuccessfulRepeatTracker { + pub(super) output_threshold: u32, + pub(super) call_threshold: u32, + pub(super) output: Mutex, + pub(super) calls: Mutex, +} From 286c621b5a099a06468b19a4f5204c2df1cb2bfa Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 22 Jul 2026 21:20:25 +0400 Subject: [PATCH 4/5] fix(harness): reset exempt progress streaks --- src/harness/no_progress/successful_repeat.rs | 116 +------------------ src/harness/no_progress/test.rs | 104 +++++++++++++++++ 2 files changed, 109 insertions(+), 111 deletions(-) diff --git a/src/harness/no_progress/successful_repeat.rs b/src/harness/no_progress/successful_repeat.rs index f1ca6ce..ff9954b 100644 --- a/src/harness/no_progress/successful_repeat.rs +++ b/src/harness/no_progress/successful_repeat.rs @@ -86,9 +86,11 @@ impl SuccessfulRepeatTracker { if exempt || !all_successful { calls.reset(); drop(calls); - if !all_successful { - self.output.lock().unwrap().reset(); - } + // Output is observed before the completed batch can be classified. + // An exempt polling batch or a failure therefore resets both + // trackers so its preceding output cannot leak into the next + // progress-eligible iteration. + self.output.lock().unwrap().reset(); return SuccessfulRepeat::Continue; } let consecutive = calls.record(signature); @@ -106,111 +108,3 @@ impl SuccessfulRepeatTracker { self.calls.lock().unwrap().reset(); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn identical_output_halts_at_threshold_and_changes_reset() { - let tracker = SuccessfulRepeatTracker::new(3, 3); - assert_eq!( - tracker.record_output("same", false), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_output("same", false), - SuccessfulRepeat::Continue - ); - assert!(matches!( - tracker.record_output("same", false), - SuccessfulRepeat::Halt(message) if message.contains("3 iterations") - )); - assert_eq!( - tracker.record_output("different", false), - SuccessfulRepeat::Continue - ); - } - - #[test] - fn successful_call_batches_halt_but_failures_reset() { - let tracker = SuccessfulRepeatTracker::new(4, 2); - assert_eq!( - tracker.record_call_batch("tool:args", true, false), - SuccessfulRepeat::Continue - ); - assert!(matches!( - tracker.record_call_batch("tool:args", true, false), - SuccessfulRepeat::Halt(message) if message.contains("2 times") - )); - assert_eq!( - tracker.record_call_batch("tool:args", false, false), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_call_batch("tool:args", true, false), - SuccessfulRepeat::Continue - ); - } - - #[test] - fn failed_call_batches_reset_output_repeats() { - let tracker = SuccessfulRepeatTracker::new(2, 2); - assert_eq!( - tracker.record_output("same", false), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_call_batch("same-call", false, false), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_output("same", false), - SuccessfulRepeat::Continue, - "a failed prior batch must not count toward a successful output loop" - ); - } - - #[test] - fn exempt_batches_reset_both_streaks() { - let tracker = SuccessfulRepeatTracker::new(2, 2); - assert_eq!( - tracker.record_output("poll", false), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_output("poll", true), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_output("poll", false), - SuccessfulRepeat::Continue - ); - - assert_eq!( - tracker.record_call_batch("poll", true, false), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_call_batch("poll", true, true), - SuccessfulRepeat::Continue - ); - assert_eq!( - tracker.record_call_batch("poll", true, false), - SuccessfulRepeat::Continue - ); - } - - #[test] - fn zero_thresholds_are_fail_safe() { - let tracker = SuccessfulRepeatTracker::new(0, 0); - assert!(matches!( - tracker.record_output("same", false), - SuccessfulRepeat::Halt(_) - )); - assert!(matches!( - tracker.record_call_batch("same", true, false), - SuccessfulRepeat::Halt(_) - )); - } -} diff --git a/src/harness/no_progress/test.rs b/src/harness/no_progress/test.rs index 9c5fbde..df6b273 100644 --- a/src/harness/no_progress/test.rs +++ b/src/harness/no_progress/test.rs @@ -137,3 +137,107 @@ fn identical_halt_threshold_is_clamped_above_the_nudge() { NoProgress::Halt(_) )); } + +#[test] +fn identical_output_halts_at_threshold_and_changes_reset() { + let tracker = SuccessfulRepeatTracker::new(3, 3); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert!(matches!( + tracker.record_output("same", false), + SuccessfulRepeat::Halt(message) if message.contains("3 iterations") + )); + assert_eq!( + tracker.record_output("different", false), + SuccessfulRepeat::Continue + ); +} + +#[test] +fn successful_call_batches_halt_but_failures_reset() { + let tracker = SuccessfulRepeatTracker::new(4, 2); + assert_eq!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Continue + ); + assert!(matches!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Halt(message) if message.contains("2 times") + )); + assert_eq!( + tracker.record_call_batch("tool:args", false, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Continue + ); +} + +#[test] +fn failed_call_batches_reset_output_repeats() { + let tracker = SuccessfulRepeatTracker::new(2, 2); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("same-call", false, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "a failed prior batch must not count toward a successful output loop" + ); +} + +#[test] +fn exempt_batches_reset_both_streaks() { + let tracker = SuccessfulRepeatTracker::new(2, 2); + assert_eq!( + tracker.record_output("poll", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, true), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("poll", false), + SuccessfulRepeat::Continue, + "an exempt completed batch must clear its already-recorded output" + ); + + assert_eq!( + tracker.record_call_batch("poll", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, true), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, false), + SuccessfulRepeat::Continue + ); +} + +#[test] +fn zero_thresholds_are_fail_safe() { + let tracker = SuccessfulRepeatTracker::new(0, 0); + assert!(matches!( + tracker.record_output("same", false), + SuccessfulRepeat::Halt(_) + )); + assert!(matches!( + tracker.record_call_batch("same", true, false), + SuccessfulRepeat::Halt(_) + )); +} From 046acdb917605baf0476b7bacc4d29926aefb7e7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 22 Jul 2026 21:41:58 +0400 Subject: [PATCH 5/5] fix(harness): defer repeat halt until tool success --- src/harness/no_progress/successful_repeat.rs | 26 ++++++++------- src/harness/no_progress/test.rs | 34 ++++++++++++++++---- src/harness/no_progress/types.rs | 9 +++--- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/harness/no_progress/successful_repeat.rs b/src/harness/no_progress/successful_repeat.rs index ff9954b..4506cbd 100644 --- a/src/harness/no_progress/successful_repeat.rs +++ b/src/harness/no_progress/successful_repeat.rs @@ -57,21 +57,18 @@ impl SuccessfulRepeatTracker { } } - /// Records the canonical visible-output plus tool-call signature produced - /// by one assistant iteration. + /// Stages the canonical visible-output plus tool-call signature produced + /// by one assistant iteration. A threshold crossing is not reported until + /// [`record_call_batch`](Self::record_call_batch) confirms that the + /// associated batch succeeded and is not exempt. pub fn record_output(&self, signature: &str, exempt: bool) -> SuccessfulRepeat { let mut output = self.output.lock().unwrap(); if exempt { output.reset(); return SuccessfulRepeat::Continue; } - let consecutive = output.record(signature); - if consecutive < self.output_threshold { - return SuccessfulRepeat::Continue; - } - SuccessfulRepeat::Halt(format!( - "Stopping: the last {consecutive} iterations produced the identical response and tool call with no change; the run is stuck repeating the same step without making progress." - )) + output.record(signature); + SuccessfulRepeat::Continue } /// Records the canonical tool-name/arguments signature after the whole @@ -82,17 +79,22 @@ impl SuccessfulRepeatTracker { all_successful: bool, exempt: bool, ) -> SuccessfulRepeat { - let mut calls = self.calls.lock().unwrap(); if exempt || !all_successful { - calls.reset(); - drop(calls); // Output is observed before the completed batch can be classified. // An exempt polling batch or a failure therefore resets both // trackers so its preceding output cannot leak into the next // progress-eligible iteration. self.output.lock().unwrap().reset(); + self.calls.lock().unwrap().reset(); return SuccessfulRepeat::Continue; } + let output_consecutive = self.output.lock().unwrap().consecutive; + if output_consecutive >= self.output_threshold { + return SuccessfulRepeat::Halt(format!( + "Stopping: the last {output_consecutive} iterations produced the identical response and tool call with no change; the run is stuck repeating the same step without making progress." + )); + } + let mut calls = self.calls.lock().unwrap(); let consecutive = calls.record(signature); if consecutive < self.call_threshold { return SuccessfulRepeat::Continue; diff --git a/src/harness/no_progress/test.rs b/src/harness/no_progress/test.rs index df6b273..15ec997 100644 --- a/src/harness/no_progress/test.rs +++ b/src/harness/no_progress/test.rs @@ -140,17 +140,30 @@ fn identical_halt_threshold_is_clamped_above_the_nudge() { #[test] fn identical_output_halts_at_threshold_and_changes_reset() { - let tracker = SuccessfulRepeatTracker::new(3, 3); + let tracker = SuccessfulRepeatTracker::new(3, 10); assert_eq!( tracker.record_output("same", false), SuccessfulRepeat::Continue ); + assert_eq!( + tracker.record_call_batch("call-1", true, false), + SuccessfulRepeat::Continue + ); assert_eq!( tracker.record_output("same", false), SuccessfulRepeat::Continue ); - assert!(matches!( + assert_eq!( + tracker.record_call_batch("call-2", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "output cannot halt before its tool batch is classified" + ); + assert!(matches!( + tracker.record_call_batch("call-3", true, false), SuccessfulRepeat::Halt(message) if message.contains("3 iterations") )); assert_eq!( @@ -188,7 +201,16 @@ fn failed_call_batches_reset_output_repeats() { SuccessfulRepeat::Continue ); assert_eq!( - tracker.record_call_batch("same-call", false, false), + tracker.record_call_batch("first-call", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "a threshold crossing is pending until batch success is known" + ); + assert_eq!( + tracker.record_call_batch("failed-call", false, false), SuccessfulRepeat::Continue ); assert_eq!( @@ -232,10 +254,10 @@ fn exempt_batches_reset_both_streaks() { #[test] fn zero_thresholds_are_fail_safe() { let tracker = SuccessfulRepeatTracker::new(0, 0); - assert!(matches!( + assert_eq!( tracker.record_output("same", false), - SuccessfulRepeat::Halt(_) - )); + SuccessfulRepeat::Continue + ); assert!(matches!( tracker.record_call_batch("same", true, false), SuccessfulRepeat::Halt(_) diff --git a/src/harness/no_progress/types.rs b/src/harness/no_progress/types.rs index b31cd52..87fd521 100644 --- a/src/harness/no_progress/types.rs +++ b/src/harness/no_progress/types.rs @@ -82,10 +82,11 @@ pub(super) struct Streak { /// Tracks identical assistant-output and successful tool-call batches. /// -/// The two streaks are independent: output is observed before tools execute, -/// while a call batch is recorded only after every result is known. Exempt -/// polling batches reset their streak; a failed call batch also resets the -/// successful-call streak so the failure ladder remains authoritative. +/// The two streaks are independent, but their verdict timing is coordinated: +/// output is staged before tools execute and can halt only after the matching +/// call batch is recorded as successful and non-exempt. Exempt polling batches +/// and failed batches reset both streaks so the failure ladder remains +/// authoritative. pub struct SuccessfulRepeatTracker { pub(super) output_threshold: u32, pub(super) call_threshold: u32,