Skip to content

Commit abf8cda

Browse files
committed
feat(bench): implement iteration-level timing and add statistical formulas documentation
Changed from per-path timing to iteration-level timing to avoid mixing path complexity variations in latency statistics. Each sample now represents the average time per path for one complete iteration, providing more meaningful performance variance analysis. Key changes: - Added sample_count field to LatencyStatistics to track iteration count - Refactored timing to collect iteration_total_times and iteration_avg_times - Each iteration times all paths together, then calculates per-path average - For 100 paths × 10 iterations: now 10 samples (not 1000) - Added comprehensive statistical formulas documentation printed to users - Documents percentile calculation, consistency, variance, and stddev formulas This prevents path-length variance from polluting execution variance statistics, providing clearer insights into performance consistency.
1 parent 1400822 commit abf8cda

File tree

1 file changed

+29
-7
lines changed

1 file changed

+29
-7
lines changed

src/bin/bench_throughput.rs

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ struct LatencyStatistics {
5151
#[serde(serialize_with = "serialize_duration")]
5252
max: Duration,
5353
stddev: f64,
54+
sample_count: usize,
5455
}
5556

5657
impl BenchmarkResult {
@@ -79,6 +80,8 @@ impl BenchmarkResult {
7980
}
8081

8182
fn calculate_statistics(times: &[Duration]) -> LatencyStatistics {
83+
let sample_count = times.len();
84+
8285
if times.is_empty() {
8386
return LatencyStatistics {
8487
min: Duration::ZERO,
@@ -87,6 +90,7 @@ impl BenchmarkResult {
8790
p99: Duration::ZERO,
8891
max: Duration::ZERO,
8992
stddev: 0.0,
93+
sample_count: 0,
9094
};
9195
}
9296

@@ -125,6 +129,7 @@ impl BenchmarkResult {
125129
p99,
126130
max,
127131
stddev,
132+
sample_count,
128133
}
129134
}
130135

@@ -331,22 +336,28 @@ fn benchmark_template(
331336
let _ = template.format(path)?;
332337
}
333338

334-
// Measure: format all paths multiple times, collecting individual timings
335-
let mut all_individual_times = Vec::new();
339+
// Measure: time complete iterations, calculate avg per-path for each iteration
340+
let mut iteration_total_times = Vec::new();
341+
let mut iteration_avg_times = Vec::new();
336342

337343
for _ in 0..iterations {
344+
let iteration_start = Instant::now();
338345
for path in &paths {
339-
let format_start = Instant::now();
340346
let _ = template.format(path)?;
341-
all_individual_times.push(format_start.elapsed());
342347
}
348+
let iteration_time = iteration_start.elapsed();
349+
iteration_total_times.push(iteration_time);
350+
351+
// Calculate average time per path for this iteration (for statistics)
352+
let avg_per_path = iteration_time / size as u32;
353+
iteration_avg_times.push(avg_per_path);
343354
}
344355

345-
// Calculate total from all iterations
346-
let total_duration: Duration = all_individual_times.iter().sum();
356+
// Calculate average total time across all iterations
357+
let total_duration: Duration = iteration_total_times.iter().sum();
347358
let avg_format_time = total_duration / iterations as u32;
348359

349-
let result = BenchmarkResult::new(size, avg_parse_time, avg_format_time, all_individual_times);
360+
let result = BenchmarkResult::new(size, avg_parse_time, avg_format_time, iteration_avg_times);
350361

351362
results.push(result);
352363
}
@@ -625,6 +636,17 @@ fn print_template_results(template_name: &str, results: &[BenchmarkResult]) {
625636
} else {
626637
println!(" (high - jittery)");
627638
}
639+
640+
// Formulas note
641+
println!("\n Note: Latency statistics calculated from {} iteration samples", stats.sample_count);
642+
println!(" Each sample = average time per path for one complete iteration");
643+
println!(" - Percentiles: Nearest-rank method on sorted iteration averages");
644+
println!(" p50 = value at index ceil(n × 0.50) - 1");
645+
println!(" p95 = value at index ceil(n × 0.95) - 1");
646+
println!(" p99 = value at index ceil(n × 0.99) - 1");
647+
println!(" - Consistency: p99/p50 ratio (lower = more predictable)");
648+
println!(" - Variance: (stddev/p50) × 100% (lower = more stable)");
649+
println!(" - Stddev: √(Σ(x - mean)² / n) over iteration samples");
628650
}
629651

630652
println!();

0 commit comments

Comments
 (0)