diff --git a/src/harness/mod.rs b/src/harness/mod.rs index 43fac9b..217eeea 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -52,7 +52,9 @@ pub use model::{ collect_model_stream, context_window_for_model_id, }; pub use no_progress::{ - DEFAULT_IDENTICAL_HALT_THRESHOLD, NoProgress, NoProgressTracker, ToolAttempt, + DEFAULT_IDENTICAL_HALT_THRESHOLD, DEFAULT_REPEAT_CALL_THRESHOLD, + DEFAULT_REPEAT_OUTPUT_THRESHOLD, NoProgress, NoProgressTracker, SuccessfulRepeat, + SuccessfulRepeatTracker, ToolAttempt, }; pub use tool::{ Tool as HarnessTool, ToolCall as HarnessToolCall, ToolFormat, ToolRegistry, diff --git a/src/harness/no_progress/mod.rs b/src/harness/no_progress/mod.rs index 981d2c8..8e98340 100644 --- a/src/harness/no_progress/mod.rs +++ b/src/harness/no_progress/mod.rs @@ -19,10 +19,14 @@ //! and turns the returned [`NoProgress`] verdict into a steering nudge //! (`Nudge`) or a halt (`Halt`). +mod successful_repeat; mod types; +pub use successful_repeat::{DEFAULT_REPEAT_CALL_THRESHOLD, DEFAULT_REPEAT_OUTPUT_THRESHOLD}; use types::LadderState; -pub use types::{NoProgress, NoProgressTracker, ToolAttempt}; +pub use types::{ + NoProgress, NoProgressTracker, SuccessfulRepeat, SuccessfulRepeatTracker, ToolAttempt, +}; use std::sync::Mutex; diff --git a/src/harness/no_progress/successful_repeat.rs b/src/harness/no_progress/successful_repeat.rs new file mode 100644 index 0000000..4506cbd --- /dev/null +++ b/src/harness/no_progress/successful_repeat.rs @@ -0,0 +1,112 @@ +//! Successful-repeat progress detection. +//! +//! [`NoProgressTracker`] handles failing tool calls, but deliberately resets on +//! success. That leaves a second loop shape undetected: a model can repeatedly +//! emit the same response and successfully invoke the same no-op tool call. +//! This tracker owns the provider- and product-neutral streak accounting for +//! those loops; a harness middleware remains responsible for building canonical +//! signatures and deciding which polling tools are exempt. + +use std::hash::{Hash, Hasher}; + +use super::types::{Streak, SuccessfulRepeat, SuccessfulRepeatTracker}; + +/// Consecutive identical assistant-output batches required to halt. +pub const DEFAULT_REPEAT_OUTPUT_THRESHOLD: u32 = 4; +/// Consecutive identical successful tool-call batches required to halt. +pub const DEFAULT_REPEAT_CALL_THRESHOLD: u32 = 3; + +impl Streak { + fn record(&mut self, signature: &str) -> u32 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + signature.hash(&mut hasher); + let hash = hasher.finish(); + if self.last_hash == Some(hash) { + self.consecutive += 1; + } else { + self.last_hash = Some(hash); + self.consecutive = 1; + } + self.consecutive + } + + fn reset(&mut self) { + *self = Self::default(); + } +} + +impl Default for SuccessfulRepeatTracker { + fn default() -> Self { + Self::new( + DEFAULT_REPEAT_OUTPUT_THRESHOLD, + DEFAULT_REPEAT_CALL_THRESHOLD, + ) + } +} + +impl SuccessfulRepeatTracker { + /// Builds a tracker. Thresholds are clamped to one so `0` cannot disable a + /// safety guard accidentally; callers that do not want this guard should + /// omit the tracker. + pub fn new(output_threshold: u32, call_threshold: u32) -> Self { + Self { + output_threshold: output_threshold.max(1), + call_threshold: call_threshold.max(1), + output: std::sync::Mutex::new(Streak::default()), + calls: std::sync::Mutex::new(Streak::default()), + } + } + + /// Stages the canonical visible-output plus tool-call signature produced + /// by one assistant iteration. A threshold crossing is not reported until + /// [`record_call_batch`](Self::record_call_batch) confirms that the + /// associated batch succeeded and is not exempt. + pub fn record_output(&self, signature: &str, exempt: bool) -> SuccessfulRepeat { + let mut output = self.output.lock().unwrap(); + if exempt { + output.reset(); + return SuccessfulRepeat::Continue; + } + output.record(signature); + SuccessfulRepeat::Continue + } + + /// Records the canonical tool-name/arguments signature after the whole + /// batch completes. Failed or exempt batches reset the successful streak. + pub fn record_call_batch( + &self, + signature: &str, + all_successful: bool, + exempt: bool, + ) -> SuccessfulRepeat { + if exempt || !all_successful { + // Output is observed before the completed batch can be classified. + // An exempt polling batch or a failure therefore resets both + // trackers so its preceding output cannot leak into the next + // progress-eligible iteration. + self.output.lock().unwrap().reset(); + self.calls.lock().unwrap().reset(); + return SuccessfulRepeat::Continue; + } + let output_consecutive = self.output.lock().unwrap().consecutive; + if output_consecutive >= self.output_threshold { + return SuccessfulRepeat::Halt(format!( + "Stopping: the last {output_consecutive} iterations produced the identical response and tool call with no change; the run is stuck repeating the same step without making progress." + )); + } + let mut calls = self.calls.lock().unwrap(); + let consecutive = calls.record(signature); + if consecutive < self.call_threshold { + return SuccessfulRepeat::Continue; + } + SuccessfulRepeat::Halt(format!( + "Stopping: the same successful tool-call batch was issued {consecutive} times in a row with identical arguments and no new information; the run is stuck repeating one action without making progress." + )) + } + + /// Clears both streaks, for example when a paused run is resumed. + pub fn reset(&self) { + self.output.lock().unwrap().reset(); + self.calls.lock().unwrap().reset(); + } +} diff --git a/src/harness/no_progress/test.rs b/src/harness/no_progress/test.rs index 9c5fbde..15ec997 100644 --- a/src/harness/no_progress/test.rs +++ b/src/harness/no_progress/test.rs @@ -137,3 +137,129 @@ fn identical_halt_threshold_is_clamped_above_the_nudge() { NoProgress::Halt(_) )); } + +#[test] +fn identical_output_halts_at_threshold_and_changes_reset() { + let tracker = SuccessfulRepeatTracker::new(3, 10); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("call-1", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("call-2", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "output cannot halt before its tool batch is classified" + ); + assert!(matches!( + tracker.record_call_batch("call-3", true, false), + SuccessfulRepeat::Halt(message) if message.contains("3 iterations") + )); + assert_eq!( + tracker.record_output("different", false), + SuccessfulRepeat::Continue + ); +} + +#[test] +fn successful_call_batches_halt_but_failures_reset() { + let tracker = SuccessfulRepeatTracker::new(4, 2); + assert_eq!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Continue + ); + assert!(matches!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Halt(message) if message.contains("2 times") + )); + assert_eq!( + tracker.record_call_batch("tool:args", false, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("tool:args", true, false), + SuccessfulRepeat::Continue + ); +} + +#[test] +fn failed_call_batches_reset_output_repeats() { + let tracker = SuccessfulRepeatTracker::new(2, 2); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("first-call", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "a threshold crossing is pending until batch success is known" + ); + assert_eq!( + tracker.record_call_batch("failed-call", false, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue, + "a failed prior batch must not count toward a successful output loop" + ); +} + +#[test] +fn exempt_batches_reset_both_streaks() { + let tracker = SuccessfulRepeatTracker::new(2, 2); + assert_eq!( + tracker.record_output("poll", false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, true), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_output("poll", false), + SuccessfulRepeat::Continue, + "an exempt completed batch must clear its already-recorded output" + ); + + assert_eq!( + tracker.record_call_batch("poll", true, false), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, true), + SuccessfulRepeat::Continue + ); + assert_eq!( + tracker.record_call_batch("poll", true, false), + SuccessfulRepeat::Continue + ); +} + +#[test] +fn zero_thresholds_are_fail_safe() { + let tracker = SuccessfulRepeatTracker::new(0, 0); + assert_eq!( + tracker.record_output("same", false), + SuccessfulRepeat::Continue + ); + assert!(matches!( + tracker.record_call_batch("same", true, false), + SuccessfulRepeat::Halt(_) + )); +} diff --git a/src/harness/no_progress/types.rs b/src/harness/no_progress/types.rs index 94c8e73..87fd521 100644 --- a/src/harness/no_progress/types.rs +++ b/src/harness/no_progress/types.rs @@ -63,3 +63,33 @@ pub struct NoProgressTracker { pub(super) identical_halt_threshold: usize, pub(super) state: Mutex, } + +/// Verdict returned after recording a successful-repeat signal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SuccessfulRepeat { + /// The signature changed, is exempt, failed, or remains below its threshold. + Continue, + /// The same successful action has repeated enough times to be considered + /// stuck. The message is suitable for steering or a halt summary. + Halt(String), +} + +#[derive(Default)] +pub(super) struct Streak { + pub(super) last_hash: Option, + pub(super) consecutive: u32, +} + +/// Tracks identical assistant-output and successful tool-call batches. +/// +/// The two streaks are independent, but their verdict timing is coordinated: +/// output is staged before tools execute and can halt only after the matching +/// call batch is recorded as successful and non-exempt. Exempt polling batches +/// and failed batches reset both streaks so the failure ladder remains +/// authoritative. +pub struct SuccessfulRepeatTracker { + pub(super) output_threshold: u32, + pub(super) call_threshold: u32, + pub(super) output: Mutex, + pub(super) calls: Mutex, +}