From 42a1fb18b686a14a270981e2cb77b678a66fd994 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 15:00:47 -0700 Subject: [PATCH 01/19] feat(tools/gasbench): compute-opcode gas-vs-execution-time microbench MVP Tracer-free differential microbenchmark: measures 22 scalar EVM opcodes (arithmetic/bitwise/comparison/stack) via core/vm/runtime.Call against a fresh in-memory StateDB, no tracer attached (per the design's load-bearing constraint: gas and time are never measured in the same run). Per-opcode gas comes from the interpreter's own accounting; per-opcode time from difference-of-medians between a balanced baseline/target bytecode pair. CoV is the first-class output and the acceptance gate (design's noise-floor criterion). Gas self-check asserts measured whole-program gas delta == definitional expected delta for every case (all 22 pass). Local integration of two coral-built components (timing/stats/emission; EVM wiring/bytecode construction) reconciled during assembly: Program (renamed from a Harness/Harness name collision) is a pre-warmed per-program executor so state setup is amortized out of the timed loop, and RunOnce is zero-arg (closes over its Program) rather than taking code per call. Local branch only, not pushed. Design: bdchatham-designs designs/gas-repricing-telemetry/gas-vs-time-instrumentation.md --- Makefile | 4 + tools/gasbench/bench_test.go | 131 ++++++++++++++++++++++++ tools/gasbench/diff.go | 68 +++++++++++++ tools/gasbench/emit.go | 75 ++++++++++++++ tools/gasbench/gasbench.go | 122 ++++++++++++++++++++++ tools/gasbench/program.go | 90 +++++++++++++++++ tools/gasbench/programs.go | 190 +++++++++++++++++++++++++++++++++++ tools/gasbench/run.sh | 48 +++++++++ tools/gasbench/stats.go | 75 ++++++++++++++ 9 files changed, 803 insertions(+) create mode 100644 tools/gasbench/bench_test.go create mode 100644 tools/gasbench/diff.go create mode 100644 tools/gasbench/emit.go create mode 100644 tools/gasbench/gasbench.go create mode 100644 tools/gasbench/program.go create mode 100644 tools/gasbench/programs.go create mode 100755 tools/gasbench/run.sh create mode 100644 tools/gasbench/stats.go diff --git a/Makefile b/Makefile index 085bf60949..398176c405 100644 --- a/Makefile +++ b/Makefile @@ -566,3 +566,7 @@ test-group-%:split-test-packages PARALLEL="-parallel=4"; \ fi; \ cat $(BUILDDIR)/packages.txt.$* | xargs go test $$PARALLEL -mod=readonly -timeout=10m -race -coverprofile=$*.profile.out -covermode=atomic -coverpkg=./... + +.PHONY: gasbench +gasbench: ## Run the compute-opcode microbench (pin core via GASBENCH_CORE) + ./tools/gasbench/run.sh diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go new file mode 100644 index 0000000000..099450bdec --- /dev/null +++ b/tools/gasbench/bench_test.go @@ -0,0 +1,131 @@ +package gasbench + +import ( + "os" + "strconv" + "testing" +) + +// BenchmarkOpcodes drives one differential measurement per scalar opcode +// Case. Run with -benchtime=1x (the inner loop is Measure's, not b.N) and +// -count=K to get K independent process runs for benchstat cross-run +// variance. +func BenchmarkOpcodes(b *testing.B) { + cases := BuildCases() + cfg := Config{ + Warmup: envInt("GASBENCH_WARMUP", 2000), + Iterations: envInt("GASBENCH_ITERS", 20000), + DisableGC: true, + LockThread: true, + } + sigmaK := envFloat("GASBENCH_SIGMA_K", 3) + covFloor := envFloat("GASBENCH_COV_FLOOR", 0.02) + + var runs []Run + for _, c := range cases { + c := c + b.Run(c.OpcodeID, func(b *testing.B) { + baseProg, err := NewProgram(c.Baseline) + if err != nil { + b.Fatalf("%s: baseline program invalid: %v", c.OpcodeID, err) + } + tgtProg, err := NewProgram(c.Target) + if err != nil { + b.Fatalf("%s: target program invalid: %v", c.OpcodeID, err) + } + + base, err := Measure(c.OpcodeID+"/baseline", baseProg.Run, cfg) + if err != nil { + b.Fatal(err) + } + tgt, err := Measure(c.OpcodeID+"/target", tgtProg.Run, cfg) + if err != nil { + b.Fatal(err) + } + + bStats := Summarize(base.Samples) + tStats := Summarize(tgt.Samples) + d := Subtract(c.OpcodeID, base, tgt, c.Reps, sigmaK, covFloor) + + // Self-check of the differential construction: the measured + // whole-program gas delta must equal the definitional expected + // delta, or the baseline/target pair is not isolating the + // opcode the way BuildCaseWith intends. This is a correctness + // check on the harness, not on opcode timing. + if d.GasDelta != c.ExpectedGasDelta { + b.Errorf("%s: gas self-check failed: measured delta %d != expected %d (baseline/target pair does not isolate the opcode)", + c.OpcodeID, d.GasDelta, c.ExpectedGasDelta) + } + + b.ReportMetric(tStats.Median, "median-ns") + b.ReportMetric(tStats.P99, "p99-ns") + b.ReportMetric(tStats.CoV, "cov") + b.ReportMetric(d.PerOpNs, "per-op-ns") + b.ReportMetric(d.PerOpGas, "per-op-gas") + + for _, w := range append(base.Warnings, tgt.Warnings...) { + b.Logf("%s: %s", c.OpcodeID, w) + } + if !d.Significant { + b.Logf("%s: delta %.1fns within noise (uncertainty %.1fns, %gσ)", + c.OpcodeID, d.DeltaNs, d.Uncertainty, sigmaK) + } + status := StatusOK + if !d.NoiseOK { + status = StatusNoisy + } + runs = append(runs, + NewRun(base, bStats, statusOf(bStats.CoV, covFloor)), + NewRun(tgt, tStats, status)) + }) + } + writeRuns(b, runs) +} + +func writeRuns(b *testing.B, runs []Run) { + if p := os.Getenv("GASBENCH_OUT_CSV"); p != "" { + f, err := os.Create(p) + if err != nil { + b.Fatalf("create csv: %v", err) + } + defer f.Close() + if err := WriteCSV(f, runs); err != nil { + b.Fatalf("write csv: %v", err) + } + } + if p := os.Getenv("GASBENCH_OUT_NDJSON"); p != "" { + f, err := os.Create(p) + if err != nil { + b.Fatalf("create ndjson: %v", err) + } + defer f.Close() + if err := WriteNDJSON(f, runs); err != nil { + b.Fatalf("write ndjson: %v", err) + } + } +} + +func statusOf(cov, floor float64) string { + if cov > floor { + return StatusNoisy + } + return StatusOK +} + +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +func envFloat(key string, def float64) float64 { + if v := os.Getenv(key); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + } + return def +} diff --git a/tools/gasbench/diff.go b/tools/gasbench/diff.go new file mode 100644 index 0000000000..2d7d1aa781 --- /dev/null +++ b/tools/gasbench/diff.go @@ -0,0 +1,68 @@ +package gasbench + +import "math" + +// Diff is the subtraction of a baseline series (opcode replaced by a +// JUMPDEST/no-op) from a target series (opcode present). Everything the two +// programs share -- loop overhead, setup, timer cost -- cancels in the mean; +// what remains is attributable to the opcode. +// +// The estimator here is difference-of-medians with the median's asymptotic +// standard error propagated in quadrature. Medians reject the additive-noise +// tail better than means. A min-difference with a bootstrap CI is the +// estimator to reach for if the central assumption proves too coarse; it is +// the honest follow-up, not the MVP. +type Diff struct { + OpcodeID string + + BaselineMedian float64 // ns + TargetMedian float64 // ns + BaselineCoV float64 + TargetCoV float64 + + DeltaNs float64 // per-program time attributable to the opcode reps + Uncertainty float64 // 1-sigma, propagated in quadrature (ns) + SigmaK float64 // significance multiple applied + Significant bool // |DeltaNs| > SigmaK*Uncertainty + + NoiseOK bool // both series CoV within the floor + CoVFloor float64 + + Reps int // opcode executions per program (from the Case) + PerOpNs float64 // DeltaNs / Reps + + GasDelta uint64 // measured whole-program gas delta; exact (ground truth) + PerOpGas float64 // GasDelta / Reps +} + +// Subtract computes the opcode cost from baseline/target series. sigmaK sets +// the significance band (e.g. 3 ~ 99.7% for a normal center); covFloor is the +// per-series noise gate; reps normalizes the per-program delta to one opcode. +func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covFloor float64) Diff { + bs := Summarize(baseline.Samples) + ts := Summarize(target.Samples) + + d := Diff{ + OpcodeID: opcodeID, + BaselineMedian: bs.Median, + TargetMedian: ts.Median, + BaselineCoV: bs.CoV, + TargetCoV: ts.CoV, + SigmaK: sigmaK, + CoVFloor: covFloor, + Reps: reps, + } + d.DeltaNs = ts.Median - bs.Median + d.Uncertainty = math.Hypot(ts.SEMedian, bs.SEMedian) + d.Significant = math.Abs(d.DeltaNs) > sigmaK*d.Uncertainty + d.NoiseOK = bs.CoV <= covFloor && ts.CoV <= covFloor + + if target.GasUsed >= baseline.GasUsed { + d.GasDelta = target.GasUsed - baseline.GasUsed + } + if reps > 0 { + d.PerOpNs = d.DeltaNs / float64(reps) + d.PerOpGas = float64(d.GasDelta) / float64(reps) + } + return d +} diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go new file mode 100644 index 0000000000..2251913bfe --- /dev/null +++ b/tools/gasbench/emit.go @@ -0,0 +1,75 @@ +package gasbench + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "strconv" +) + +// Run is one emitted measurement row. +type Run struct { + InputID string `json:"input_id"` + GasUsed uint64 `json:"gas_used"` + ExecTimeNs int64 `json:"exec_time_ns"` // median sample + Status string `json:"status"` + Iterations int `json:"iterations"` + CoV float64 `json:"cov"` +} + +// Status values. +const ( + StatusOK = "ok" + StatusNoisy = "noisy" // CoV above floor: measurement not trustworthy + StatusError = "error" +) + +// NewRun builds a Run from a measured series; status is chosen by the caller +// after applying the CoV gate. +func NewRun(s Series, st Stats, status string) Run { + return Run{ + InputID: s.InputID, + GasUsed: s.GasUsed, + ExecTimeNs: int64(st.Median), + Status: status, + Iterations: st.N, + CoV: st.CoV, + } +} + +var csvHeader = []string{"input_id", "gas_used", "exec_time_ns", "status", "iterations", "cov"} + +// WriteCSV writes a header plus one row per run. +func WriteCSV(w io.Writer, runs []Run) error { + cw := csv.NewWriter(w) + if err := cw.Write(csvHeader); err != nil { + return err + } + for _, r := range runs { + rec := []string{ + r.InputID, + strconv.FormatUint(r.GasUsed, 10), + strconv.FormatInt(r.ExecTimeNs, 10), + r.Status, + strconv.Itoa(r.Iterations), + strconv.FormatFloat(r.CoV, 'g', 6, 64), + } + if err := cw.Write(rec); err != nil { + return err + } + } + cw.Flush() + return cw.Error() +} + +// WriteNDJSON writes one JSON object per line. +func WriteNDJSON(w io.Writer, runs []Run) error { + enc := json.NewEncoder(w) // Encode appends a newline: NDJSON by construction + for i := range runs { + if err := enc.Encode(&runs[i]); err != nil { + return fmt.Errorf("gasbench: encode run %d: %w", i, err) + } + } + return nil +} diff --git a/tools/gasbench/gasbench.go b/tools/gasbench/gasbench.go new file mode 100644 index 0000000000..0b86775076 --- /dev/null +++ b/tools/gasbench/gasbench.go @@ -0,0 +1,122 @@ +// Package gasbench measures per-opcode EVM execution time and correlates it +// with gas cost via differential microbenchmarking. +// +// The harness is tracer-free by construction: nothing here attaches an EVM +// tracer, so the interpreter runs its hot path unobserved. Timing uses the +// monotonic clock (time.Now/time.Since) around a caller-supplied RunOnce. +package gasbench + +import ( + "fmt" + "runtime" + "runtime/debug" + "time" +) + +// RunOnce executes a pre-loaded EVM program to completion through the +// tracer-free interpreter and reports the gas it consumed. +// +// A RunOnce closes over its program (see Program.Run in program.go) so the +// timing loop never rebuilds interpreter/StateDB state per call. Rebuilding +// per call would swamp a nanosecond-scale opcode signal with allocation +// noise; the differential construction only holds if baseline and target pay +// identical, amortized-out setup cost. +// +// Contract the EVM-wiring side must satisfy: +// - Deterministic: identical program yields identical gasUsed and +// equivalent work every call; no state carried across calls that would +// drift the timing (Program.Run resets transient storage/access list +// per entry, which is safe only for pure-compute, non-state-touching +// programs). +// - Self-contained and tracer-free: no I/O, logging, or tracer on the hot +// path; the only thing measured is interpreter execution. +// - Sufficient work: the program runs the opcode-under-test enough times +// that total runtime is well above timer resolution (target >= ~10us), +// so per-call time.Now overhead and clock granularity do not dominate. +// - Allocation-light: GC is disabled during the window, so allocations +// across warmup+Iterations must fit in RAM. Keep the program lean or +// lower Iterations. +type RunOnce func() (gasUsed uint64, err error) + +// sink defeats dead-code elimination of the gas result. A single measurement +// goroutine (GOMAXPROCS=1, locked thread) means no synchronization is needed. +var sink uint64 + +// Config controls one measurement series. +type Config struct { + Warmup int // discarded iterations; warm I-cache/D-cache/branch predictor and lazy init + Iterations int // timed iterations retained as samples + DisableGC bool // stop the collector for the measurement window (recommended) + LockThread bool // pin the goroutine to its OS thread for the window (recommended) +} + +// DefaultConfig is a sane starting point; tune Iterations to the noise floor. +func DefaultConfig() Config { + return Config{Warmup: 2000, Iterations: 20000, DisableGC: true, LockThread: true} +} + +// Series is the raw output of one measurement: per-iteration wall-clock samples +// plus the deterministic gas cost. +type Series struct { + InputID string + GasUsed uint64 + Samples []time.Duration + Warnings []string +} + +// Measure runs Warmup discarded iterations, then Iterations timed iterations +// of run(), returning per-iteration monotonic samples. +func Measure(id string, run RunOnce, cfg Config) (Series, error) { + if run == nil { + return Series{}, fmt.Errorf("gasbench: nil RunOnce for %q", id) + } + if cfg.Iterations < 1 { + return Series{}, fmt.Errorf("gasbench: Iterations must be >= 1 for %q", id) + } + + s := Series{InputID: id, Samples: make([]time.Duration, cfg.Iterations)} + if n := runtime.GOMAXPROCS(0); n != 1 { + s.Warnings = append(s.Warnings, + fmt.Sprintf("GOMAXPROCS=%d (want 1): scheduler noise will inflate the tail", n)) + } + + if cfg.LockThread { + // Stop the OS from migrating this goroutine across cores mid-window, + // which would cost cold-cache and cross-NUMA restarts inside a sample. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + } + + // Warmup, discarded. Also the first place a broken program is caught. + var g uint64 + var err error + for i := 0; i < cfg.Warmup; i++ { + if g, err = run(); err != nil { + return Series{}, fmt.Errorf("gasbench: warmup failed for %q: %w", id, err) + } + } + sink ^= g + + if cfg.DisableGC { + // A GC pause landing inside a sample is pure tail noise and is exactly + // what corrupts p99. Collect once to start from a clean heap, stop the + // collector for the window, and restore the prior target on exit. Cost: + // the heap grows unbounded during the window (see RunOnce contract). + runtime.GC() + prev := debug.SetGCPercent(-1) + defer debug.SetGCPercent(prev) + } + + for i := 0; i < cfg.Iterations; i++ { + start := time.Now() + g, err = run() + d := time.Since(start) // monotonic; immune to wall-clock/NTP steps + if err != nil { + return Series{}, fmt.Errorf("gasbench: iteration %d failed for %q: %w", i, id, err) + } + s.Samples[i] = d + sink ^= g + } + s.GasUsed = g + return s, nil +} diff --git a/tools/gasbench/program.go b/tools/gasbench/program.go new file mode 100644 index 0000000000..db011ab0d1 --- /dev/null +++ b/tools/gasbench/program.go @@ -0,0 +1,90 @@ +package gasbench + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/runtime" +) + +// contractAddr matches the address runtime.Execute uses internally, for +// parity with the fork's own test harness. +var contractAddr = common.BytesToAddress([]byte("contract")) + +// newRuntimeConfig builds a fresh, tracer-free interpreter config against a +// fresh in-memory StateDB. +// +// This does NOT use runtime.Execute. Execute discards leftOverGas (it only +// surfaces gas via a tracer's OnTxEnd hook, and we run tracer-free - +// EVMConfig.Tracer == nil, the debug=false path), so it cannot return gas. +// runtime.Call is the identical fresh-EVM, tracer-free code path but returns +// leftOverGas directly. We replicate Execute's fresh-state setup +// (state.New + CreateAccount + SetCode) so the "fresh in-memory StateDB, +// stateless" property this MVP relies on is preserved. +func newRuntimeConfig(code []byte) (*runtime.Config, error) { + cfg := new(runtime.Config) + runtime.SetDefaults(cfg) // Shanghai+Cancun active, GasLimit=MaxUint64, no tracer + cfg.EVMConfig = vm.Config{} + + sdb, err := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + if err != nil { + return nil, fmt.Errorf("gasbench: new statedb: %w", err) + } + cfg.State = sdb + cfg.State.CreateAccount(contractAddr) + cfg.State.SetCode(contractAddr, code) + return cfg, nil +} + +// Program holds a warmed EVM environment so a timing loop measures only the +// hot Call, not StateDB construction (which would swamp a nanosecond-scale +// opcode signal). Build one Program per bytecode input; call Run in the +// timing loop. +// +// Reusing a single StateDB across Run calls is safe here BECAUSE the +// benchmark programs are pure compute (no SSTORE/LOG/CREATE): they make no +// persistent state change, and runtime.Call resets transient storage and the +// access list on every entry via statedb.Prepare. Do not reuse a Program for +// state-touching code. +type Program struct { + cfg *runtime.Config +} + +// NewProgram loads code into a warmed environment and validates that it runs +// clean once before any timing. A program that reverts/OOGs is rejected here +// so the timing loop never records a bogus sample. +func NewProgram(code []byte) (*Program, error) { + cfg, err := newRuntimeConfig(code) + if err != nil { + return nil, err + } + p := &Program{cfg: cfg} + if _, err := p.Run(); err != nil { + return nil, fmt.Errorf("gasbench: program does not run clean: %w", err) + } + return p, nil +} + +// Run executes the loaded code once, returning EVM gas consumed. This is the +// hot function for the timing loop: it does no per-call state allocation. +// +// gasUsed is cfg.GasLimit - leftOverGas: total EVM gas the interpreter +// charged. There is deliberately NO Cosmos/Sei gas meter in this path - +// runtime.Call builds a bare go-ethereum EVM with no ante handler and no +// sei-cosmos GasMeter. Isolating the EVM interpreter from the Cosmos layer is +// the entire point of the stateless-compute MVP. +// +// err is nil on a clean STOP/RETURN. On REVERT err is vm.ErrExecutionReverted +// (gasUsed is partial - unused gas is refunded on revert). On out-of-gas err +// is vm.ErrOutOfGas (gasUsed == GasLimit). On a bad/undefined opcode err is +// vm.ErrInvalidOpCode. The differential programs built in programs.go +// terminate cleanly (STOP, balanced stack), so a non-nil err means the run +// is INVALID - the caller must discard the sample, never treat it as a +// measurement. Matches the gasbench.RunOnce contract (gasbench.go). +func (p *Program) Run() (gasUsed uint64, err error) { + _, leftOverGas, callErr := runtime.Call(contractAddr, nil, p.cfg) + return p.cfg.GasLimit - leftOverGas, callErr +} diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go new file mode 100644 index 0000000000..0b3285f66f --- /dev/null +++ b/tools/gasbench/programs.go @@ -0,0 +1,190 @@ +package gasbench + +import ( + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" + "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" +) + +// DefaultReps is the number of unrolled body units per program. Straight-line +// (no loop counter) keeps the interpreter dispatch loop identical between +// baseline and target; the count is fixed and equal for both, so loop-control +// cost cannot leak into the differential. 1000 gives a strong signal while +// staying well under any code/gas limit. +const DefaultReps = 1000 + +// Class groups opcodes by arithmetic character. +type Class string + +const ( + ClassArithmetic Class = "arithmetic" + ClassBitwise Class = "bitwise" + ClassComparison Class = "comparison" + ClassStack Class = "stack" +) + +// OpSpec is one benchmarkable opcode. +type OpSpec struct { + Name string + Op vm.OpCode + Class Class + Arity int // stack items consumed by the op (operands) + ConstGas uint64 // definitional constant gas from the fork jump table + // DataDependent is true when execution TIME varies with operand + // magnitude (uint256 limb-count loops) even though gas is constant. + // Treat these as curves, not points, if the harness ever sweeps operands + // (deferred - see the design's Non-goals). + DataDependent bool +} + +// Specs is the benchmarkable scalar-opcode set. Every entry is CONSTANT-gas +// in this fork (verified against core/vm/jump_table.go + core/vm/eips.go); +// none has dynamic gas, none touches memory/state. EXP is deliberately +// omitted: its gas is parametric (gasExpFrontier over exponent byte length), +// so it does not belong in a constant-gas differential. +// +// No scalar opcode here is repriced on Sei: production always builds the EVM +// with a stock vm.Config{} (x/evm/keeper/evm.go, msg_server.go, ante/fee.go), +// no custom jump table, no opcode gas overrides. The only Sei gas param is +// SeiSstoreSetGasEIP2200 (x/evm/types/params.go), a storage opcode and out of +// scope here. If a scalar opcode is ever repriced, its gas must come from the +// live x/evm params, not this table's geth constant. +var Specs = []OpSpec{ + // arithmetic + {"ADD", vm.ADD, ClassArithmetic, 2, vm.GasFastestStep, false}, + {"MUL", vm.MUL, ClassArithmetic, 2, vm.GasFastStep, true}, + {"SUB", vm.SUB, ClassArithmetic, 2, vm.GasFastestStep, false}, + {"DIV", vm.DIV, ClassArithmetic, 2, vm.GasFastStep, true}, + {"MOD", vm.MOD, ClassArithmetic, 2, vm.GasFastStep, true}, + {"ADDMOD", vm.ADDMOD, ClassArithmetic, 3, vm.GasMidStep, true}, + {"MULMOD", vm.MULMOD, ClassArithmetic, 3, vm.GasMidStep, true}, + // bitwise + {"AND", vm.AND, ClassBitwise, 2, vm.GasFastestStep, false}, + {"OR", vm.OR, ClassBitwise, 2, vm.GasFastestStep, false}, + {"XOR", vm.XOR, ClassBitwise, 2, vm.GasFastestStep, false}, + {"NOT", vm.NOT, ClassBitwise, 1, vm.GasFastestStep, false}, + {"SHL", vm.SHL, ClassBitwise, 2, vm.GasFastestStep, true}, + {"SHR", vm.SHR, ClassBitwise, 2, vm.GasFastestStep, true}, + {"SAR", vm.SAR, ClassBitwise, 2, vm.GasFastestStep, true}, + // comparison + {"LT", vm.LT, ClassComparison, 2, vm.GasFastestStep, false}, + {"GT", vm.GT, ClassComparison, 2, vm.GasFastestStep, false}, + {"SLT", vm.SLT, ClassComparison, 2, vm.GasFastestStep, false}, + {"SGT", vm.SGT, ClassComparison, 2, vm.GasFastestStep, false}, + {"EQ", vm.EQ, ClassComparison, 2, vm.GasFastestStep, false}, + {"ISZERO", vm.ISZERO, ClassComparison, 1, vm.GasFastestStep, false}, + // stack + {"DUP1", vm.DUP1, ClassStack, 0, vm.GasFastestStep, false}, + {"SWAP1", vm.SWAP1, ClassStack, 0, vm.GasFastestStep, false}, +} + +// Case is a differential bytecode pair for one opcode, ready for measurement. +// Baseline and target are identical except that target executes the opcode; +// both run the same fixed unit count and both terminate cleanly on STOP with +// a balanced stack. +// +// ExpectedGasDelta is the definitional, whole-program gas the opcode adds +// (per-unit delta * Reps) - NOT necessarily Reps*ConstGas, because a net-0, +// cleanly-looping body needs stack rebalancing (see BuildPairWith). It is the +// expected value; the measured Diff.GasDelta from a live run is the ground +// truth, and the two should be cross-checked (see bench_test.go) as a +// self-check of the differential construction itself. +type Case struct { + OpcodeID string + Class Class + DataDependent bool + Reps int + Baseline []byte + Target []byte + ExpectedGasDelta uint64 +} + +// seed256 is the fixed operand pushed once as the working value. Full-width +// (2^256-1) so the magnitude-dependent ops (MUL/DIV/MOD/ADDMOD/MULMOD, +// shifts) exercise their full uint256 limb path rather than a small-number +// fast case. Exposed via BuildCaseWith for operand sweeps. +var seed256 = new(uint256.Int).Not(uint256.NewInt(0)) + +// BuildCases builds every spec's case at DefaultReps. +func BuildCases() []Case { + out := make([]Case, 0, len(Specs)) + for _, s := range Specs { + out = append(out, BuildCaseWith(s, DefaultReps, seed256)) + } + return out +} + +// BuildCaseWith constructs the differential case for one opcode. +// +// Construction (the balanced-DUP/POP differential, the fork's +// benchmarkNonModifyingCode technique adapted to a bounded, clean-terminating +// straight line). For an op consuming n operands and producing 1 result, the +// net-0 repeating unit is: +// +// target = DUP1 x n , OP , POP // dup n copies of top, run op, drop result +// baseline = DUP1 x n , POP x n // dup n copies of top, drop them +// +// The n DUP1s are identical on both sides and cancel; one POP cancels the +// target's trailing POP. What remains -- the whole differential -- is the op +// vs (n-1) extra POPs, so the per-unit gas delta = ConstGas(op) - (n-1)*GasQuickStep. +// - n=1 (NOT, ISZERO): delta = ConstGas(op) (op isolated exactly) +// - n=2 (most ops): delta = ConstGas(op) - 2 +// - n=3 (ADDMOD/MULMOD): delta = ConstGas(op) - 4 +// +// Stack ops are special-cased (not "n operands -> 1 result"): +// - DUP1 (1->2): target = DUP1 POP ; baseline = PUSH0 POP -> delta = 3-2 = 1 +// - SWAP1 (2->2): target = SWAP1 ; baseline = JUMPDEST -> delta = 3-1 = 2 +// +// A single seed value is pushed as the prologue; every unit is net-0 so stack +// depth never grows (no 1024-depth risk) and never underflows. +func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { + base := program.New() + tgt := program.New() + + // Prologue: seed the stack. Two copies so SWAP1 has two items and any op + // always has a live top to DUP. + base.Push(seed).Push(seed) + tgt.Push(seed).Push(seed) + + var perUnitDelta uint64 + switch s.Op { + case vm.DUP1: + for i := 0; i < reps; i++ { + tgt.Op(vm.DUP1, vm.POP) + base.Op(vm.PUSH0, vm.POP) + } + perUnitDelta = s.ConstGas - vm.GasQuickStep // 3 - 2 + case vm.SWAP1: + for i := 0; i < reps; i++ { + tgt.Op(vm.SWAP1) + base.Op(vm.JUMPDEST) + } + perUnitDelta = s.ConstGas - params.JumpdestGas // 3 - 1 + default: + for i := 0; i < reps; i++ { + for d := 0; d < s.Arity; d++ { + tgt.Op(vm.DUP1) + base.Op(vm.DUP1) + } + tgt.Op(s.Op, vm.POP) + for d := 0; d < s.Arity; d++ { + base.Op(vm.POP) + } + } + perUnitDelta = s.ConstGas - uint64(s.Arity-1)*vm.GasQuickStep + } + + base.Op(vm.STOP) + tgt.Op(vm.STOP) + + return Case{ + OpcodeID: s.Name, + Class: s.Class, + DataDependent: s.DataDependent, + Reps: reps, + Baseline: base.Bytes(), + Target: tgt.Bytes(), + ExpectedGasDelta: perUnitDelta * uint64(reps), + } +} diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh new file mode 100755 index 0000000000..09f3dbc190 --- /dev/null +++ b/tools/gasbench/run.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +# One measurement thread pinned to one core. Set CORE to a core that is NOT +# shared with an active hyperthread sibling and, ideally, isolated on the +# kernel cmdline (isolcpus/nohz_full/rcu_nocbs). +CORE="${GASBENCH_CORE:-3}" + +# --- Operator responsibilities the PROCESS CANNOT set ----------------------- +# These MUST be set at OS/BIOS before trusting numbers. This runner checks and +# warns; it cannot enforce them. On a dedicated EC2 host, ensure no co-tenants. +# * Turbo/boost OFF: echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo +# (or disable in BIOS) -- else frequency drifts per-sample. +# * Governor=performance: +# sudo cpupower frequency-set -g performance +# * Deep C-states OFF: boot intel_idle.max_cstate=1 processor.max_cstate=1 +# (C-state exit latency lands in the tail). +# * SMT OFF for the measured core, or pin away from its sibling. +# * Core isolation: isolcpus=$CORE nohz_full=$CORE rcu_nocbs=$CORE (cmdline). +# * IRQ affinity moved off $CORE. + +warn() { printf 'WARN: %s\n' "$*" >&2; } + +if [ -r /sys/devices/system/cpu/intel_pstate/no_turbo ]; then + [ "$(cat /sys/devices/system/cpu/intel_pstate/no_turbo)" = "1" ] || warn "turbo appears ENABLED" +fi +gov="/sys/devices/system/cpu/cpu${CORE}/cpufreq/scaling_governor" +if [ -r "$gov" ]; then + [ "$(cat "$gov")" = "performance" ] || warn "cpu${CORE} governor != performance" +fi + +export GOMAXPROCS=1 +export GASBENCH_ITERS="${GASBENCH_ITERS:-20000}" +export GASBENCH_WARMUP="${GASBENCH_WARMUP:-2000}" +export GASBENCH_COV_FLOOR="${GASBENCH_COV_FLOOR:-0.02}" +export GASBENCH_OUT_CSV="${GASBENCH_OUT_CSV:-gasbench.csv}" +export GASBENCH_OUT_NDJSON="${GASBENCH_OUT_NDJSON:-gasbench.ndjson}" + +PIN=(env) +if command -v taskset >/dev/null 2>&1; then + PIN=(taskset -c "$CORE") +else + warn "taskset not found (non-Linux?); running unpinned -- results are indicative only" +fi + +exec "${PIN[@]}" go test ./tools/gasbench/ \ + -run '^$' -bench '^BenchmarkOpcodes$' \ + -benchtime=1x -count="${GASBENCH_COUNT:-10}" -benchmem diff --git a/tools/gasbench/stats.go b/tools/gasbench/stats.go new file mode 100644 index 0000000000..ccc291b92d --- /dev/null +++ b/tools/gasbench/stats.go @@ -0,0 +1,75 @@ +package gasbench + +import ( + "math" + "sort" + "time" +) + +// Stats summarizes a sample series. All time fields are nanoseconds. +type Stats struct { + N int + Min float64 // least-perturbed estimator for CPU-bound work; noise only adds time + Max float64 + Mean float64 + Median float64 + P99 float64 + Stddev float64 // sample stddev (n-1) + CoV float64 // Stddev/Mean; the acceptance gate ("noise floor") + SEMean float64 // standard error of the mean + SEMedian float64 // asymptotic standard error of the median +} + +// Summarize computes the full stat set over the samples. +func Summarize(samples []time.Duration) Stats { + n := len(samples) + st := Stats{N: n} + if n == 0 { + return st + } + xs := make([]float64, n) + var sum float64 + for i, d := range samples { + xs[i] = float64(d.Nanoseconds()) + sum += xs[i] + } + sort.Float64s(xs) + + st.Min = xs[0] + st.Max = xs[n-1] + st.Mean = sum / float64(n) + st.Median = percentile(xs, 50) + st.P99 = percentile(xs, 99) + + if n > 1 { + var ss float64 + for _, x := range xs { + dx := x - st.Mean + ss += dx * dx + } + st.Stddev = math.Sqrt(ss / float64(n-1)) + st.SEMean = st.Stddev / math.Sqrt(float64(n)) + // SE of the median for a roughly-normal center: sqrt(pi/2)*SEMean. + st.SEMedian = 1.2533141373155 * st.SEMean + } + if st.Mean > 0 { + st.CoV = st.Stddev / st.Mean + } + return st +} + +// percentile interpolates linearly between closest ranks (numpy default). +// xs must be sorted ascending. +func percentile(xs []float64, p float64) float64 { + n := len(xs) + if n == 1 { + return xs[0] + } + rank := p / 100 * float64(n-1) + lo := int(math.Floor(rank)) + hi := int(math.Ceil(rank)) + if lo == hi { + return xs[lo] + } + return xs[lo] + (rank-float64(lo))*(xs[hi]-xs[lo]) +} From a2542710f01c0b891e5896d2116728d8a2b194ca Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 15:18:03 -0700 Subject: [PATCH 02/19] =?UTF-8?q?fix(tools/gasbench):=20resolve=20xreview?= =?UTF-8?q?=20R1=20dissent=20=E2=80=94=20significance=20gate,=20persisted?= =?UTF-8?q?=20diff,=20shift-op=20operands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three correctness-grade findings from systems-engineer (assigned dissenter): - MISMATCH: Run.Status was gated only on NoiseOK (per-series stability), never on Diff.Significant (whether the delta clears measurement uncertainty). A statistically-insignificant opcode delta could be emitted Status="ok". Status is now OK only when both NoiseOK and Significant hold, with a new StatusInsignificant distinguishing 'noisy measurement' from 'stable series, zero-indistinguishable delta'. - MISSING: persisted CSV/NDJSON carried only the raw per-series median (one row per baseline/target variant), never the differential itself. Run is redesigned to one row per opcode carrying Reps, GasUsed (the measured whole-program gas delta), ExecTimeNs (the measured whole- program time delta), and Significant -- the actual (gas, time) pair the design's acceptance criteria describe. - MISMATCH: SHL/SHR/SAR reused the same full-width seed as both stack operands, making the shift AMOUNT itself 2^256-1 -- go-ethereum's value.Clear() early-out for shifts >=256, the cheapest possible path, not the limb-shift work the old comment claimed to measure. BuildCaseWith now special-cases the shift family with a distinct in-range shift-amount operand (seedShift) via a DUP2/DUP2 construction that keeps the differential net-0. Verified: post-fix per-op-ns for SHL/SHR/SAR rose ~50-70% (1.3-1.4ns -> 2.0-2.4ns), consistent with exercising real limb-shift work instead of the degenerate clear path. Also closes two non-blocking items from the other two lenses: - solidity-developer: Program's reuse-safety doc comment cited 'no state change' as the reason it's safe to reuse across calls; corrected to the real basis (deterministic gas + tail-only, baseline/target- symmetric journal growth from an un-Finalise-d Snapshot per call), with a warning against extending to high-iteration or state-touching use without re-checking. - idiomatic-reviewer (style/advisory): DefaultConfig() is now the actual source of truth bench_test.go reads through (was dead code, values triplicated in run.sh); WriteCSV errors now wrap with the gasbench: prefix like WriteNDJSON; Case.Class now flows all the way to the persisted Run/CSV row instead of being unused; Series.InputID and StatusError documented. All 22 opcodes re-verified: gas self-check passes, build/vet/gofmt clean. Local branch only, not pushed. --- tools/gasbench/bench_test.go | 46 +++++++++---------- tools/gasbench/emit.go | 88 ++++++++++++++++++++++++++---------- tools/gasbench/gasbench.go | 2 +- tools/gasbench/program.go | 19 ++++++-- tools/gasbench/programs.go | 52 +++++++++++++++++---- tools/gasbench/run.sh | 6 +-- 6 files changed, 148 insertions(+), 65 deletions(-) diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index 099450bdec..cd2a4c5949 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -12,11 +12,12 @@ import ( // variance. func BenchmarkOpcodes(b *testing.B) { cases := BuildCases() + def := DefaultConfig() cfg := Config{ - Warmup: envInt("GASBENCH_WARMUP", 2000), - Iterations: envInt("GASBENCH_ITERS", 20000), - DisableGC: true, - LockThread: true, + Warmup: envInt("GASBENCH_WARMUP", def.Warmup), + Iterations: envInt("GASBENCH_ITERS", def.Iterations), + DisableGC: def.DisableGC, + LockThread: def.LockThread, } sigmaK := envFloat("GASBENCH_SIGMA_K", 3) covFloor := envFloat("GASBENCH_COV_FLOOR", 0.02) @@ -43,8 +44,6 @@ func BenchmarkOpcodes(b *testing.B) { b.Fatal(err) } - bStats := Summarize(base.Samples) - tStats := Summarize(tgt.Samples) d := Subtract(c.OpcodeID, base, tgt, c.Reps, sigmaK, covFloor) // Self-check of the differential construction: the measured @@ -52,31 +51,37 @@ func BenchmarkOpcodes(b *testing.B) { // delta, or the baseline/target pair is not isolating the // opcode the way BuildCaseWith intends. This is a correctness // check on the harness, not on opcode timing. + // + // Note: this check is insensitive to a mis-transcribed + // OpSpec.Arity that still yields a self-consistent (if wrong) + // ExpectedGasDelta -- it catches a wrong ConstGas, a wrong + // opcode, or unexpected dynamic/memory gas, not an arity error. + // All 22 Specs entries were hand-verified against the fork's + // jump-table minStack/maxStack; verify a future addition the + // same way. if d.GasDelta != c.ExpectedGasDelta { b.Errorf("%s: gas self-check failed: measured delta %d != expected %d (baseline/target pair does not isolate the opcode)", c.OpcodeID, d.GasDelta, c.ExpectedGasDelta) } - b.ReportMetric(tStats.Median, "median-ns") - b.ReportMetric(tStats.P99, "p99-ns") - b.ReportMetric(tStats.CoV, "cov") + b.ReportMetric(d.BaselineMedian, "baseline-median-ns") + b.ReportMetric(d.TargetMedian, "target-median-ns") b.ReportMetric(d.PerOpNs, "per-op-ns") b.ReportMetric(d.PerOpGas, "per-op-gas") for _, w := range append(base.Warnings, tgt.Warnings...) { b.Logf("%s: %s", c.OpcodeID, w) } + if !d.NoiseOK { + b.Logf("%s: series CoV above floor %.4g (baseline=%.4g target=%.4g) -- measurement not trustworthy", + c.OpcodeID, covFloor, d.BaselineCoV, d.TargetCoV) + } if !d.Significant { - b.Logf("%s: delta %.1fns within noise (uncertainty %.1fns, %gσ)", + b.Logf("%s: delta %.1fns within noise (uncertainty %.1fns, %gσ) -- marginal cost not distinguishable from zero at this precision", c.OpcodeID, d.DeltaNs, d.Uncertainty, sigmaK) } - status := StatusOK - if !d.NoiseOK { - status = StatusNoisy - } - runs = append(runs, - NewRun(base, bStats, statusOf(bStats.CoV, covFloor)), - NewRun(tgt, tStats, status)) + + runs = append(runs, NewRun(c, d, cfg.Iterations)) }) } writeRuns(b, runs) @@ -105,13 +110,6 @@ func writeRuns(b *testing.B, runs []Run) { } } -func statusOf(cov, floor float64) string { - if cov > floor { - return StatusNoisy - } - return StatusOK -} - func envInt(key string, def int) int { if v := os.Getenv(key); v != "" { if n, err := strconv.Atoi(v); err == nil { diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index 2251913bfe..e9718f4ce4 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -8,59 +8,101 @@ import ( "strconv" ) -// Run is one emitted measurement row. +// Run is one emitted measurement row: the per-opcode differential result, +// ready for a gas-vs-time correlation. This is the record the design's +// acceptance criteria describe (per-input gas_used and exec_time_ns, +// supporting both a point lookup and a cross-input regression) -- for this +// harness "input" is the opcode, and the values that answer "does gas track +// time" are the differential (target-minus-baseline) gas and time, not +// either series' raw median. type Run struct { - InputID string `json:"input_id"` - GasUsed uint64 `json:"gas_used"` - ExecTimeNs int64 `json:"exec_time_ns"` // median sample - Status string `json:"status"` - Iterations int `json:"iterations"` - CoV float64 `json:"cov"` + InputID string `json:"input_id"` // opcode id, e.g. "ADD" + Class string `json:"class"` // opcode family, e.g. "arithmetic" + Reps int `json:"reps"` // opcode executions the delta represents + GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps + ExecTimeNs float64 `json:"exec_time_ns"` // whole-program time delta, ns (target median - baseline median); per-op = ExecTimeNs/Reps + Status string `json:"status"` // ok only if both series clear the noise floor AND the delta is statistically significant + Iterations int `json:"iterations"` // timed iterations behind each series + CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV + Significant bool `json:"significant"` // |delta| exceeds SigmaK * propagated uncertainty } // Status values. const ( StatusOK = "ok" - StatusNoisy = "noisy" // CoV above floor: measurement not trustworthy + StatusNoisy = "noisy" // a series' CoV is above the floor: measurement not trustworthy + // StatusInsignificant marks a case whose series are individually stable + // (NoiseOK) but whose delta does not clear measurement uncertainty -- + // the opcode's marginal cost may be real but is not distinguishable from + // zero at this precision. Distinct from StatusNoisy: re-running a noisy + // case may help, re-running an insignificant one on the same host will + // not -- it needs more Iterations or a coarser SigmaK, not a quieter host. + StatusInsignificant = "insignificant" + // StatusError is reserved for a per-case measurement failure. Not + // currently produced: bench_test.go fails the whole benchmark loudly + // (b.Fatalf) on an invalid program rather than degrading to an error + // row, so every emitted Run today is OK/Noisy/Insignificant. StatusError = "error" ) -// NewRun builds a Run from a measured series; status is chosen by the caller -// after applying the CoV gate. -func NewRun(s Series, st Stats, status string) Run { +// NewRun builds a Run from a differential result. c is the Case that +// produced d (for its Class); iterations is the sample count behind each +// series (baseline and target share one Config, so a single value applies +// to both). +func NewRun(c Case, d Diff, iterations int) Run { + cov := d.BaselineCoV + if d.TargetCoV > cov { + cov = d.TargetCoV + } + status := StatusOK + switch { + case !d.NoiseOK: + status = StatusNoisy + case !d.Significant: + status = StatusInsignificant + } return Run{ - InputID: s.InputID, - GasUsed: s.GasUsed, - ExecTimeNs: int64(st.Median), - Status: status, - Iterations: st.N, - CoV: st.CoV, + InputID: d.OpcodeID, + Class: string(c.Class), + Reps: d.Reps, + GasUsed: d.GasDelta, + ExecTimeNs: d.DeltaNs, + Status: status, + Iterations: iterations, + CoV: cov, + Significant: d.Significant, } } -var csvHeader = []string{"input_id", "gas_used", "exec_time_ns", "status", "iterations", "cov"} +var csvHeader = []string{"input_id", "class", "reps", "gas_used", "exec_time_ns", "status", "iterations", "cov", "significant"} // WriteCSV writes a header plus one row per run. func WriteCSV(w io.Writer, runs []Run) error { cw := csv.NewWriter(w) if err := cw.Write(csvHeader); err != nil { - return err + return fmt.Errorf("gasbench: write csv header: %w", err) } - for _, r := range runs { + for i, r := range runs { rec := []string{ r.InputID, + r.Class, + strconv.Itoa(r.Reps), strconv.FormatUint(r.GasUsed, 10), - strconv.FormatInt(r.ExecTimeNs, 10), + strconv.FormatFloat(r.ExecTimeNs, 'g', -1, 64), r.Status, strconv.Itoa(r.Iterations), strconv.FormatFloat(r.CoV, 'g', 6, 64), + strconv.FormatBool(r.Significant), } if err := cw.Write(rec); err != nil { - return err + return fmt.Errorf("gasbench: write csv row %d: %w", i, err) } } cw.Flush() - return cw.Error() + if err := cw.Error(); err != nil { + return fmt.Errorf("gasbench: flush csv: %w", err) + } + return nil } // WriteNDJSON writes one JSON object per line. diff --git a/tools/gasbench/gasbench.go b/tools/gasbench/gasbench.go index 0b86775076..018378f66d 100644 --- a/tools/gasbench/gasbench.go +++ b/tools/gasbench/gasbench.go @@ -58,7 +58,7 @@ func DefaultConfig() Config { // Series is the raw output of one measurement: per-iteration wall-clock samples // plus the deterministic gas cost. type Series struct { - InputID string + InputID string // the measured input's identity, e.g. "ADD/baseline" (opcode + variant) GasUsed uint64 Samples []time.Duration Warnings []string diff --git a/tools/gasbench/program.go b/tools/gasbench/program.go index db011ab0d1..ce1c58a46a 100644 --- a/tools/gasbench/program.go +++ b/tools/gasbench/program.go @@ -44,11 +44,20 @@ func newRuntimeConfig(code []byte) (*runtime.Config, error) { // opcode signal). Build one Program per bytecode input; call Run in the // timing loop. // -// Reusing a single StateDB across Run calls is safe here BECAUSE the -// benchmark programs are pure compute (no SSTORE/LOG/CREATE): they make no -// persistent state change, and runtime.Call resets transient storage and the -// access list on every entry via statedb.Prepare. Do not reuse a Program for -// state-touching code. +// Reusing a single StateDB across many Run calls is safe for these +// pure-compute programs, but not because "no state change occurs": every +// call's evm.Call takes a Snapshot() that is never reverted or Finalise-d +// (Finalise is what resets the journal; this harness never calls it), so the +// journal's revision/touch-change list grows roughly two entries per call +// across the life of a Program. That growth is harmless here because (a) gas +// accounting is unaffected -- it comes only from interpreter opcodes -- and +// (b) the growth is symmetric between baseline and target (both take exactly +// one Snapshot + one zero-value Transfer per call) and lands as tail latency +// from periodic slice reallocation, not a shift in the median the +// difference-of-medians estimator reads. Do NOT assume this still holds if +// Iterations grows by an order of magnitude, or if a future Case touches +// persistent state (SSTORE/LOG/CREATE) -- re-check the journal-growth impact +// before reusing a Program across a much longer or state-touching series. type Program struct { cfg *runtime.Config } diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index 0b3285f66f..f0852dbd74 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -101,11 +101,21 @@ type Case struct { } // seed256 is the fixed operand pushed once as the working value. Full-width -// (2^256-1) so the magnitude-dependent ops (MUL/DIV/MOD/ADDMOD/MULMOD, -// shifts) exercise their full uint256 limb path rather than a small-number -// fast case. Exposed via BuildCaseWith for operand sweeps. +// (2^256-1) so the magnitude-dependent ops (MUL/DIV/MOD/ADDMOD/MULMOD) +// exercise their full uint256 limb path rather than a small-number fast +// case. Exposed via BuildCaseWith for operand sweeps. var seed256 = new(uint256.Int).Not(uint256.NewInt(0)) +// seedShift is a fixed, in-range (<256) shift amount for SHL/SHR/SAR. A +// shift amount >= 256 takes go-ethereum's value.Clear() early-out (the +// degenerate, cheapest case: the result is zero without touching the +// value's limbs), so reusing seed256 as BOTH operands -- as the generic +// same-value construction below does for every other opcode -- measures +// that early-out, not representative shift work. Kept separate from the +// value operand so the shift ops still exercise the interpreter's limb-shift +// path on a full-width value. +var seedShift = uint256.NewInt(4) + // BuildCases builds every spec's case at DefaultReps. func BuildCases() []Case { out := make([]Case, 0, len(Specs)) @@ -136,32 +146,56 @@ func BuildCases() []Case { // - DUP1 (1->2): target = DUP1 POP ; baseline = PUSH0 POP -> delta = 3-2 = 1 // - SWAP1 (2->2): target = SWAP1 ; baseline = JUMPDEST -> delta = 3-1 = 2 // -// A single seed value is pushed as the prologue; every unit is net-0 so stack +// SHL/SHR/SAR are also special-cased: they need two DISTINCT operands (see +// seedShift), not "n copies of the same value". The prologue pushes +// (seed, seedShift); DUP2 reaches two positions down without disturbing the +// other operand, so the SAME (value, shift) pair is reused net-0 across +// every unit: +// +// target = DUP2, DUP2, OP , POP // dup shift then value, run op, drop result +// baseline = DUP2, DUP2, POP, POP // dup shift then value, drop both +// +// This yields the same arity=2 shape as the general case: +// delta = ConstGas(op) - GasQuickStep (one extra POP in baseline vs target). +// +// Every case's prologue seeds its own stack, and every unit is net-0 so stack // depth never grows (no 1024-depth risk) and never underflows. func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { base := program.New() tgt := program.New() - // Prologue: seed the stack. Two copies so SWAP1 has two items and any op - // always has a live top to DUP. - base.Push(seed).Push(seed) - tgt.Push(seed).Push(seed) - var perUnitDelta uint64 switch s.Op { case vm.DUP1: + base.Push(seed).Push(seed) + tgt.Push(seed).Push(seed) for i := 0; i < reps; i++ { tgt.Op(vm.DUP1, vm.POP) base.Op(vm.PUSH0, vm.POP) } perUnitDelta = s.ConstGas - vm.GasQuickStep // 3 - 2 + case vm.SWAP1: + base.Push(seed).Push(seed) + tgt.Push(seed).Push(seed) for i := 0; i < reps; i++ { tgt.Op(vm.SWAP1) base.Op(vm.JUMPDEST) } perUnitDelta = s.ConstGas - params.JumpdestGas // 3 - 1 + + case vm.SHL, vm.SHR, vm.SAR: + base.Push(seed).Push(seedShift) + tgt.Push(seed).Push(seedShift) + for i := 0; i < reps; i++ { + tgt.Op(vm.DUP2, vm.DUP2, s.Op, vm.POP) + base.Op(vm.DUP2, vm.DUP2, vm.POP, vm.POP) + } + perUnitDelta = s.ConstGas - vm.GasQuickStep // 3 - 2, same shape as the n=2 default case + default: + base.Push(seed).Push(seed) + tgt.Push(seed).Push(seed) for i := 0; i < reps; i++ { for d := 0; d < s.Arity; d++ { tgt.Op(vm.DUP1) diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh index 09f3dbc190..0a60d874a6 100755 --- a/tools/gasbench/run.sh +++ b/tools/gasbench/run.sh @@ -30,9 +30,9 @@ if [ -r "$gov" ]; then fi export GOMAXPROCS=1 -export GASBENCH_ITERS="${GASBENCH_ITERS:-20000}" -export GASBENCH_WARMUP="${GASBENCH_WARMUP:-2000}" -export GASBENCH_COV_FLOOR="${GASBENCH_COV_FLOOR:-0.02}" +# GASBENCH_ITERS / GASBENCH_WARMUP / GASBENCH_COV_FLOOR are left unset here by +# default: gasbench.DefaultConfig() in Go is the single source of truth for +# those defaults. Set them in the environment only to override for this run. export GASBENCH_OUT_CSV="${GASBENCH_OUT_CSV:-gasbench.csv}" export GASBENCH_OUT_NDJSON="${GASBENCH_OUT_NDJSON:-gasbench.ndjson}" From 6652cea372fb2119e9a3d9cd11c98520b129a179 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 15:24:18 -0700 Subject: [PATCH 03/19] docs(tools/gasbench): fix run.sh comment accuracy per xreview R2 (idiomatic-reviewer) Co-Authored-By: Claude Fable 5 --- tools/gasbench/run.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh index 0a60d874a6..811318090c 100755 --- a/tools/gasbench/run.sh +++ b/tools/gasbench/run.sh @@ -30,9 +30,11 @@ if [ -r "$gov" ]; then fi export GOMAXPROCS=1 -# GASBENCH_ITERS / GASBENCH_WARMUP / GASBENCH_COV_FLOOR are left unset here by -# default: gasbench.DefaultConfig() in Go is the single source of truth for -# those defaults. Set them in the environment only to override for this run. +# GASBENCH_ITERS / GASBENCH_WARMUP are left unset here by default: +# gasbench.DefaultConfig() in Go is the single source of truth for those two. +# GASBENCH_COV_FLOOR / GASBENCH_SIGMA_K default in bench_test.go instead (no +# Config field for them). Set any of these in the environment only to +# override for this run. export GASBENCH_OUT_CSV="${GASBENCH_OUT_CSV:-gasbench.csv}" export GASBENCH_OUT_NDJSON="${GASBENCH_OUT_NDJSON:-gasbench.ndjson}" From 27205a7c7e53809db60b01223ed03b044aa389b3 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 16:44:09 -0700 Subject: [PATCH 04/19] refactor(tools/gasbench): re-prioritize acceptance gate per noise-isolation research Three refinements derived from designs/gas-repricing-telemetry/research/ microbenchmark-noise-isolation-tradeoffs.md: - Status is now purely Significant-driven (ok/insignificant). CoV no longer overrides a statistically solid result -- three independent benchmarking harnesses (JMH, criterion.rs, Go's own benchstat) all gate on effect-size-vs-uncertainty, not a fixed dispersion threshold, and the research's practitioner sweep gave the same reasoning (CoV describes sample dispersion, not confidence in the derived median; the differential's own propagated SE is the right metric). CoV survives as an independent HighVariance advisory flag. - Recalibrated the default health-check ceiling from 2% to 25%: above the 4-8% CoV measured as normal on a dedicated, pinned, bare-metal host with no kernel-level isolation, well below the "something is actually wrong" territory (~40%+) the research's practitioner sweep named. - Added getrusage-based active-benchmarking diagnostics (Nvcsw/Nivcsw deltas around the timed window). This automates, on every future run, the same active-benchmarking check (Gregg's methodology) that was previously a one-off manual perf-stat/interrupts session on the EC2 host -- involuntary context-switches are now the direct, measured evidence behind HighVariance, not merely inferred from CoV. Deliberately NOT adding opportunistic cpuset/IRQ-affinity isolation to run.sh -- the research's explicit recommendation was not to invest further isolation effort for this MVP; doing so now would contradict that finding. Local branch only, not pushed. --- tools/gasbench/bench_test.go | 21 +++++++--- tools/gasbench/diff.go | 45 ++++++++++++++++++---- tools/gasbench/emit.go | 75 +++++++++++++++++++----------------- tools/gasbench/gasbench.go | 42 ++++++++++++++++++++ tools/gasbench/run.sh | 3 +- 5 files changed, 136 insertions(+), 50 deletions(-) diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index cd2a4c5949..ada12fbe5d 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -20,7 +20,14 @@ func BenchmarkOpcodes(b *testing.B) { LockThread: def.LockThread, } sigmaK := envFloat("GASBENCH_SIGMA_K", 3) - covFloor := envFloat("GASBENCH_COV_FLOOR", 0.02) + // 0.25: an advisory health-check ceiling, not the acceptance gate (that's + // Significant). Set well above the several-percent CoV plain core-pinning + // sees in practice (measured 4-8% on a dedicated bare-metal host with no + // kernel-level isolation), so it only fires on something genuinely + // pathological -- a noisy neighbor, a throttling event -- not routine + // scheduler-tick/IRQ noise. See + // designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md. + covCeiling := envFloat("GASBENCH_COV_CEILING", 0.25) var runs []Run for _, c := range cases { @@ -44,7 +51,7 @@ func BenchmarkOpcodes(b *testing.B) { b.Fatal(err) } - d := Subtract(c.OpcodeID, base, tgt, c.Reps, sigmaK, covFloor) + d := Subtract(c.OpcodeID, base, tgt, c.Reps, sigmaK, covCeiling) // Self-check of the differential construction: the measured // whole-program gas delta must equal the definitional expected @@ -72,14 +79,16 @@ func BenchmarkOpcodes(b *testing.B) { for _, w := range append(base.Warnings, tgt.Warnings...) { b.Logf("%s: %s", c.OpcodeID, w) } - if !d.NoiseOK { - b.Logf("%s: series CoV above floor %.4g (baseline=%.4g target=%.4g) -- measurement not trustworthy", - c.OpcodeID, covFloor, d.BaselineCoV, d.TargetCoV) - } + // Significant is the acceptance gate; HighVariance is advisory + // only and never overrides it (see emit.go, diff.go doc comments). if !d.Significant { b.Logf("%s: delta %.1fns within noise (uncertainty %.1fns, %gσ) -- marginal cost not distinguishable from zero at this precision", c.OpcodeID, d.DeltaNs, d.Uncertainty, sigmaK) } + if d.HighVariance { + b.Logf("%s: series CoV above health-check ceiling %.4g (baseline=%.4g target=%.4g, nivcsw base=%d tgt=%d) -- worth investigating the host, does not invalidate a significant result", + c.OpcodeID, covCeiling, d.BaselineCoV, d.TargetCoV, base.NivcswDelta, tgt.NivcswDelta) + } runs = append(runs, NewRun(c, d, cfg.Iterations)) }) diff --git a/tools/gasbench/diff.go b/tools/gasbench/diff.go index 2d7d1aa781..19d4b5ee0d 100644 --- a/tools/gasbench/diff.go +++ b/tools/gasbench/diff.go @@ -12,6 +12,15 @@ import "math" // tail better than means. A min-difference with a bootstrap CI is the // estimator to reach for if the central assumption proves too coarse; it is // the honest follow-up, not the MVP. +// +// Acceptance is Significant, not CoV. A raw per-series CoV of several percent +// under plain core-pinning (no kernel-level isolcpus/nohz_full/rcu_nocbs) is +// expected physics -- the periodic scheduler tick, device IRQs, and +// shared-cache contention that pinning alone cannot remove -- not a defect. +// No major benchmarking harness (JMH, criterion.rs, Go's own benchstat) gates +// on a fixed dispersion threshold either; all three gate on effect size +// against the estimate's own uncertainty, which is what Significant computes. +// See designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md. type Diff struct { OpcodeID string @@ -23,10 +32,24 @@ type Diff struct { DeltaNs float64 // per-program time attributable to the opcode reps Uncertainty float64 // 1-sigma, propagated in quadrature (ns) SigmaK float64 // significance multiple applied - Significant bool // |DeltaNs| > SigmaK*Uncertainty + Significant bool // |DeltaNs| > SigmaK*Uncertainty -- the acceptance gate (see emit.go's Status) + + // HighVariance is an advisory-only flag: true when either series' CoV + // exceeds CoVCeiling. It does not gate Status; it exists to catch a + // genuinely pathological run (a noisy neighbor, a throttling event) -- + // distinct from the routine several-percent CoV a pinned-but-not-isolated + // core sees, which HighVariance's default ceiling is set well above. + HighVariance bool + CoVCeiling float64 - NoiseOK bool // both series CoV within the floor - CoVFloor float64 + // BaselineNivcsw/TargetNivcsw are involuntary-context-switch counts + // observed during each series' timed window (process-wide; see + // Series.NivcswDelta). Nonzero values are direct evidence the kernel + // preempted the measurement thread mid-window -- the mechanism behind + // HighVariance and behind ordinary CoV, made observable rather than + // merely inferred. + BaselineNivcsw int64 + TargetNivcsw int64 Reps int // opcode executions per program (from the Case) PerOpNs float64 // DeltaNs / Reps @@ -36,9 +59,13 @@ type Diff struct { } // Subtract computes the opcode cost from baseline/target series. sigmaK sets -// the significance band (e.g. 3 ~ 99.7% for a normal center); covFloor is the -// per-series noise gate; reps normalizes the per-program delta to one opcode. -func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covFloor float64) Diff { +// the significance band (e.g. 3 ~ 99.7% for a normal center) and is the +// acceptance gate. covCeiling is an advisory health-check ceiling, not a +// gate: a raw per-series CoV above it flags the run for a closer look +// (a possible neighbor or throttling event) without invalidating an +// otherwise-significant result. reps normalizes the per-program delta to one +// opcode. +func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covCeiling float64) Diff { bs := Summarize(baseline.Samples) ts := Summarize(target.Samples) @@ -49,13 +76,15 @@ func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covFlo BaselineCoV: bs.CoV, TargetCoV: ts.CoV, SigmaK: sigmaK, - CoVFloor: covFloor, + CoVCeiling: covCeiling, + BaselineNivcsw: baseline.NivcswDelta, + TargetNivcsw: target.NivcswDelta, Reps: reps, } d.DeltaNs = ts.Median - bs.Median d.Uncertainty = math.Hypot(ts.SEMedian, bs.SEMedian) d.Significant = math.Abs(d.DeltaNs) > sigmaK*d.Uncertainty - d.NoiseOK = bs.CoV <= covFloor && ts.CoV <= covFloor + d.HighVariance = bs.CoV > covCeiling || ts.CoV > covCeiling if target.GasUsed >= baseline.GasUsed { d.GasDelta = target.GasUsed - baseline.GasUsed diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index e9718f4ce4..63b8c239c5 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -16,32 +16,32 @@ import ( // time" are the differential (target-minus-baseline) gas and time, not // either series' raw median. type Run struct { - InputID string `json:"input_id"` // opcode id, e.g. "ADD" - Class string `json:"class"` // opcode family, e.g. "arithmetic" - Reps int `json:"reps"` // opcode executions the delta represents - GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps - ExecTimeNs float64 `json:"exec_time_ns"` // whole-program time delta, ns (target median - baseline median); per-op = ExecTimeNs/Reps - Status string `json:"status"` // ok only if both series clear the noise floor AND the delta is statistically significant - Iterations int `json:"iterations"` // timed iterations behind each series - CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV - Significant bool `json:"significant"` // |delta| exceeds SigmaK * propagated uncertainty + InputID string `json:"input_id"` // opcode id, e.g. "ADD" + Class string `json:"class"` // opcode family, e.g. "arithmetic" + Reps int `json:"reps"` // opcode executions the delta represents + GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps + ExecTimeNs float64 `json:"exec_time_ns"` // whole-program time delta, ns (target median - baseline median); per-op = ExecTimeNs/Reps + Status string `json:"status"` // ok if the delta is statistically significant; insignificant otherwise. NOT gated on CoV -- see HighVariance. + Iterations int `json:"iterations"` // timed iterations behind each series + CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV -- informational, not the acceptance gate + Significant bool `json:"significant"` // |delta| exceeds SigmaK * propagated uncertainty -- the acceptance gate + HighVariance bool `json:"high_variance"` // advisory: CoV exceeded the health-check ceiling; worth a look, does not invalidate Status=ok + Nivcsw int64 `json:"nivcsw"` // worse (max) of the baseline/target involuntary-context-switch count; 0 means undisturbed by scheduler preemption } // Status values. +// +// Acceptance is Significant, not CoV: a raw per-series CoV of several +// percent under plain core-pinning is expected physics, not a defect (see +// designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md). +// CoV rides as the advisory HighVariance flag instead of gating Status. const ( - StatusOK = "ok" - StatusNoisy = "noisy" // a series' CoV is above the floor: measurement not trustworthy - // StatusInsignificant marks a case whose series are individually stable - // (NoiseOK) but whose delta does not clear measurement uncertainty -- - // the opcode's marginal cost may be real but is not distinguishable from - // zero at this precision. Distinct from StatusNoisy: re-running a noisy - // case may help, re-running an insignificant one on the same host will - // not -- it needs more Iterations or a coarser SigmaK, not a quieter host. - StatusInsignificant = "insignificant" + StatusOK = "ok" // Significant is true -- the delta is trustworthy regardless of routine CoV + StatusInsignificant = "insignificant" // Significant is false -- the delta does not clear its own measurement uncertainty; needs more Iterations or a coarser SigmaK, not a quieter host // StatusError is reserved for a per-case measurement failure. Not // currently produced: bench_test.go fails the whole benchmark loudly // (b.Fatalf) on an invalid program rather than degrading to an error - // row, so every emitted Run today is OK/Noisy/Insignificant. + // row, so every emitted Run today is OK or Insignificant. StatusError = "error" ) @@ -54,27 +54,30 @@ func NewRun(c Case, d Diff, iterations int) Run { if d.TargetCoV > cov { cov = d.TargetCoV } - status := StatusOK - switch { - case !d.NoiseOK: - status = StatusNoisy - case !d.Significant: - status = StatusInsignificant + nivcsw := d.BaselineNivcsw + if d.TargetNivcsw > nivcsw { + nivcsw = d.TargetNivcsw + } + status := StatusInsignificant + if d.Significant { + status = StatusOK } return Run{ - InputID: d.OpcodeID, - Class: string(c.Class), - Reps: d.Reps, - GasUsed: d.GasDelta, - ExecTimeNs: d.DeltaNs, - Status: status, - Iterations: iterations, - CoV: cov, - Significant: d.Significant, + InputID: d.OpcodeID, + Class: string(c.Class), + Reps: d.Reps, + GasUsed: d.GasDelta, + ExecTimeNs: d.DeltaNs, + Status: status, + Iterations: iterations, + CoV: cov, + Significant: d.Significant, + HighVariance: d.HighVariance, + Nivcsw: nivcsw, } } -var csvHeader = []string{"input_id", "class", "reps", "gas_used", "exec_time_ns", "status", "iterations", "cov", "significant"} +var csvHeader = []string{"input_id", "class", "reps", "gas_used", "exec_time_ns", "status", "iterations", "cov", "significant", "high_variance", "nivcsw"} // WriteCSV writes a header plus one row per run. func WriteCSV(w io.Writer, runs []Run) error { @@ -93,6 +96,8 @@ func WriteCSV(w io.Writer, runs []Run) error { strconv.Itoa(r.Iterations), strconv.FormatFloat(r.CoV, 'g', 6, 64), strconv.FormatBool(r.Significant), + strconv.FormatBool(r.HighVariance), + strconv.FormatInt(r.Nivcsw, 10), } if err := cw.Write(rec); err != nil { return fmt.Errorf("gasbench: write csv row %d: %w", i, err) diff --git a/tools/gasbench/gasbench.go b/tools/gasbench/gasbench.go index 018378f66d..edff0316a9 100644 --- a/tools/gasbench/gasbench.go +++ b/tools/gasbench/gasbench.go @@ -10,6 +10,7 @@ import ( "fmt" "runtime" "runtime/debug" + "syscall" "time" ) @@ -62,6 +63,31 @@ type Series struct { GasUsed uint64 Samples []time.Duration Warnings []string + + // NvcswDelta/NivcswDelta are process-wide (RUSAGE_SELF, not thread-scoped + // -- Go exposes no portable RUSAGE_THREAD) voluntary/involuntary + // context-switch counts observed during the timed window (Warmup + // excluded). This is the cheapest "active benchmarking" check available + // (Gregg): a nonzero NivcswDelta on a nominally pinned core is direct + // confirmation the kernel scheduler preempted the measurement thread + // mid-window -- the same tick/IRQ/neighbor mechanism a CoV reading only + // infers indirectly. See + // designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md. + NvcswDelta int64 + NivcswDelta int64 +} + +// rusageSnapshot reads the process's current voluntary/involuntary +// context-switch counters. Process-wide rather than thread-scoped, so on a +// process with other active goroutines the delta can include their +// scheduling activity too; for this harness's single-goroutine measurement +// loop that's a minor overcount, not a different mechanism. +func rusageSnapshot() (nvcsw, nivcsw int64, err error) { + var ru syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { + return 0, 0, fmt.Errorf("gasbench: getrusage: %w", err) + } + return int64(ru.Nvcsw), int64(ru.Nivcsw), nil } // Measure runs Warmup discarded iterations, then Iterations timed iterations @@ -107,6 +133,12 @@ func Measure(id string, run RunOnce, cfg Config) (Series, error) { defer debug.SetGCPercent(prev) } + nvcswBefore, nivcswBefore, rerr := rusageSnapshot() + rusageOK := rerr == nil + if rerr != nil { + s.Warnings = append(s.Warnings, rerr.Error()) + } + for i := 0; i < cfg.Iterations; i++ { start := time.Now() g, err = run() @@ -118,5 +150,15 @@ func Measure(id string, run RunOnce, cfg Config) (Series, error) { sink ^= g } s.GasUsed = g + + if rusageOK { + nvcswAfter, nivcswAfter, rerr := rusageSnapshot() + if rerr != nil { + s.Warnings = append(s.Warnings, rerr.Error()) + } else { + s.NvcswDelta = nvcswAfter - nvcswBefore + s.NivcswDelta = nivcswAfter - nivcswBefore + } + } return s, nil } diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh index 811318090c..1841dce800 100755 --- a/tools/gasbench/run.sh +++ b/tools/gasbench/run.sh @@ -32,7 +32,8 @@ fi export GOMAXPROCS=1 # GASBENCH_ITERS / GASBENCH_WARMUP are left unset here by default: # gasbench.DefaultConfig() in Go is the single source of truth for those two. -# GASBENCH_COV_FLOOR / GASBENCH_SIGMA_K default in bench_test.go instead (no +# GASBENCH_COV_CEILING (an advisory health-check, not the acceptance gate -- +# see emit.go/diff.go) / GASBENCH_SIGMA_K default in bench_test.go instead (no # Config field for them). Set any of these in the environment only to # override for this run. export GASBENCH_OUT_CSV="${GASBENCH_OUT_CSV:-gasbench.csv}" From 611545c92f250f9823945b27c08127d7f45e6f52 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 17:05:29 -0700 Subject: [PATCH 05/19] refactor(tools/gasbench): close xreview R1 findings, move rationale to docs Closes the findings from this refinement's first xreview round (systems-engineer, idiomatic-reviewer, solidity-developer): - Fixed stats.go's stale "CoV is the acceptance gate" comment (contradicted the refinement's own thesis). - Corrected the Nvcsw/Nivcsw doc comments: RUSAGE_SELF is process-wide, not thread-scoped, and a nonzero Nivcsw explains only the scheduler-preemption slice of CoV, not the Program-reuse journal-growth tail (a separate, already-documented noise source). - Surfaced the previously write-only-and-dead Nvcsw counter through to Diff and Run, and into the CSV/NDJSON schema (new nvcsw column). - Fixed bench_test.go's false claim that -count=K gives K independent process runs (Go reruns within the same OS process). - Added a table test (emit_test.go) pinning that Status is a pure function of Significant, independent of HighVariance/CoV. Also restructures where rationale lives, per direct feedback: this package's doc comments had grown into multi-paragraph methodology essays (the differential-construction algebra, the acceptance-gate research, the active-benchmarking diagnostics). Moved that narrative into tools/gasbench/README.md and tools/gasbench/AGENTS.md; code comments are back to lean, present-state statements that point at the docs for why. Wired tools/gasbench/AGENTS.md into the top-level AGENTS.md's nested-guides list. Local branch only, not pushed. --- AGENTS.md | 1 + tools/gasbench/AGENTS.md | 47 ++++++++ tools/gasbench/README.md | 207 +++++++++++++++++++++++++++++++++++ tools/gasbench/bench_test.go | 30 ++--- tools/gasbench/diff.go | 44 +++----- tools/gasbench/emit.go | 32 +++--- tools/gasbench/emit_test.go | 30 +++++ tools/gasbench/gasbench.go | 54 +++------ tools/gasbench/program.go | 54 +++------ tools/gasbench/programs.go | 100 ++++------------- tools/gasbench/run.sh | 7 +- tools/gasbench/stats.go | 2 +- 12 files changed, 376 insertions(+), 232 deletions(-) create mode 100644 tools/gasbench/AGENTS.md create mode 100644 tools/gasbench/README.md create mode 100644 tools/gasbench/emit_test.go diff --git a/AGENTS.md b/AGENTS.md index 4bc1538058..cafbe986e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,7 @@ progressively the deeper you go. Existing package guides include: - `evmrpc/AGENTS.md` — EVM JSON-RPC (`eth_*`, `sei_*`, `sei2_*`, `debug_*`) semantics - `x/evm/AGENTS.md` — EVM module: address association, StateDB bridge, precompiles, pointers - `sei-tendermint/AGENTS.md` — sei-tendermint module conventions +- `tools/gasbench/AGENTS.md` — per-opcode gas-vs-time differential microbenchmark harness ## Code style diff --git a/tools/gasbench/AGENTS.md b/tools/gasbench/AGENTS.md new file mode 100644 index 0000000000..667f79807b --- /dev/null +++ b/tools/gasbench/AGENTS.md @@ -0,0 +1,47 @@ +# gasbench + +Differential microbenchmark harness: per-opcode EVM execution time vs gas +cost, tracer-free. See `README.md` for the full rationale (differential +construction, acceptance-gate design, active-benchmarking diagnostics); this +file is the short orientation. + +## Rules for changes here + +- **Never attach a tracer to a timed run.** `debug=true` in the interpreter + dilates per-step time — gas and time are always measured in separate, + tracer-free runs. See README.md. +- **Correlate against EVM gas, never the Cosmos gas meter.** `Program.Run` + deliberately has no ante handler / `GasMeter` in its path. +- **A new `Case` must terminate cleanly** (STOP, balanced stack, net-zero + gas per unit) — `NewProgram` rejects anything that doesn't run clean once, + and `bench_test.go`'s self-check will fail loudly if the algebra is wrong. +- **New opcode specs:** hand-verify `Arity`/`ConstGas` against the fork's + `core/vm/jump_table.go` + `core/vm/eips.go`; the self-check catches a wrong + `ConstGas` but not a self-consistent wrong `Arity`. `ConstGas` must come + from the geth constant, not a Sei override — production never reprices a + scalar opcode (stock `vm.Config{}`, no custom jump table); the only Sei gas + param is `SeiSstoreSetGasEIP2200`, a storage opcode and out of scope here. +- Keep code comments lean (what, not why); put methodology/rationale in + README.md instead of growing doc comments. + +## Running + +```bash +tools/gasbench/run.sh +``` + +Env vars, output schema, and the `-count` semantics caveat: see README.md +"Running it" / "Output schema". + +## Files + +| File | Contents | +|---|---| +| `gasbench.go` | timing core: `Measure`, `Config`, `Series`, rusage snapshot | +| `diff.go` | `Subtract`: baseline/target differencing, `Diff`, the acceptance gate | +| `stats.go` | `Summarize`: median/stddev/CoV/standard-error over a sample series | +| `program.go` | `Program`: warmed tracer-free EVM environment, one bytecode input | +| `programs.go` | `OpSpec`/`Specs`/`Case`/`BuildCaseWith`: the differential bytecode construction | +| `emit.go` | `Run`, CSV/NDJSON output | +| `bench_test.go` | `BenchmarkOpcodes`: wires the above into `go test -bench` | +| `run.sh` | pinned-core runner + operator checklist for turbo/governor/isolation | diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md new file mode 100644 index 0000000000..1e30ee9175 --- /dev/null +++ b/tools/gasbench/README.md @@ -0,0 +1,207 @@ +# gasbench + +Differential microbenchmark harness that measures per-opcode EVM execution +time and correlates it with gas cost. Built to unblock the gas-repricing +project's F2 hypothesis test: does gas cost track execution time. + +Design: `designs/gas-repricing-telemetry/gas-vs-time-instrumentation.md` in +the `bdchatham-designs` repo. Noise-floor calibration behind the acceptance +gate below: `designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md` +in the same repo. + +## The load-bearing invariant: never measure gas and time in the same run + +Attaching a `tracing.Hooks` tracer to sum per-opcode gas sets `debug=true` in +the interpreter loop, which dilates per-step execution time. So: + +- **Time** comes from a tracer-free run (`vm.Config{}`, nil `Tracer` — + production's own hot path). +- **Gas** is deterministic and comes from a separate, tracer-free run too + (`GasLimit - leftOverGas` via `runtime.Call`, not `runtime.Execute`, which + discards `leftOverGas`). + +Both series in a `Case` (baseline and target) are measured this way, so +gas and time are never read off the same call. + +## Differential construction + +Each `Case` (`programs.go`) is a baseline/target bytecode pair, identical +except the target executes the opcode under test. Everything the two +programs share — loop overhead, setup, dispatch cost — cancels when +`Subtract` (`diff.go`) takes the difference; what remains is attributable to +the opcode. + +The repeating unit is balanced so gas is net-zero and the stack never grows: +for an opcode consuming `n` operands and producing 1 result, + +``` +target = DUP1 x n , OP , POP // dup n copies, run op, drop result +baseline = DUP1 x n , POP x n // dup n copies, drop them +``` + +The `n` DUP1s cancel, one POP cancels the target's trailing POP, so the +per-unit gas delta is `ConstGas(op) - (n-1)*GasQuickStep`. Two opcode +families are special-cased in `BuildCaseWith`: + +- **Stack ops** (`DUP1`, `SWAP1`) aren't "n operands → 1 result" — each gets + its own construction. +- **SHL/SHR/SAR** need two *distinct* operands, not `n` copies of the same + value: a shift amount ≥256 takes go-ethereum's `value.Clear()` early-out + (the cheapest possible case), so reusing one seed for both operands would + measure that early-out instead of the real limb-shift path. `seedShift` + keeps the shift amount in-range and distinct from the value operand. + +`EXP` and anything with dynamic/memory/state gas is out of scope — see +`OpSpec.DataDependent` and the design's Non-goals. + +## Gas isolation from the Cosmos layer + +`Program.Run` builds a bare go-ethereum EVM (`runtime.Call`) with no ante +handler and no Sei `GasMeter`. `gasUsed` is pure EVM gas +(`cfg.GasLimit - leftOverGas`); there is deliberately no Cosmos gas meter in +this path. Correlate against **EVM** gas, not the Cosmos gas meter — mixing +the two units was the CON-368 trap (`msg_server.go:152`). + +## Program reuse across calls (journal growth) + +A `Program` (`program.go`) is built once per bytecode input and its `Run` +method is called once per timed iteration, reusing one `StateDB` — rebuilding +state per call would swamp a nanosecond-scale signal with allocation noise. +Each call takes a `Snapshot()` that's never reverted or `Finalise`-d, so the +journal's revision/touch-change list grows roughly two entries per call over +a `Program`'s life. This is harmless for the current MVP: growth is +symmetric between baseline and target (both take one snapshot + one +zero-value transfer per call), gas accounting is unaffected (it comes only +from interpreter opcodes), and the cost lands as periodic tail latency from +slice reallocation, not a shift in the median the difference-of-medians +estimator reads. **Re-check this before reusing a `Program` across an +order-of-magnitude more iterations, or for a `Case` that touches persistent +state (SSTORE/LOG/CREATE).** + +This periodic reallocation is also the one CoV-inflating noise source the +`getrusage` diagnostics below structurally cannot see (it's not a scheduler +preemption). + +## Error contract + +`Program.Run` returns `err == nil` only on a clean STOP/RETURN. `REVERT`, +out-of-gas, and invalid-opcode all return a non-nil `err` (see the doc +comment on `Run` for the specific sentinel per case). Every `Case` built by +`BuildCaseWith` terminates cleanly, so a non-nil `err` during measurement +means the run is invalid — `Measure` (`gasbench.go`) propagates it as a hard +error rather than recording a bogus sample. + +## Measurement methodology + +`Measure` (`gasbench.go`) runs `Warmup` discarded iterations (warm +I-cache/D-cache/branch predictor, catch a broken program early), then +`Iterations` timed iterations of `RunOnce`, config in `Config`/`DefaultConfig`: + +- GC disabled for the timed window (a GC pause inside a sample corrupts the + tail) — the heap grows unbounded during the window, so keep the program + allocation-light or lower `Iterations`. +- The measuring goroutine is locked to its OS thread and `GOMAXPROCS` should + be 1, so nothing migrates the goroutine across cores mid-window. +- Timing uses `time.Now`/`time.Since` (monotonic, immune to NTP steps) + around the tracer-free `RunOnce` call. + +## Acceptance gate: Significant, not CoV + +`Diff.Significant` (`diff.go`) — `|DeltaNs| > SigmaK * Uncertainty`, where +`Uncertainty` is the two series' median standard errors propagated in +quadrature — is the acceptance gate. A raw per-series CoV of several percent +under plain core-pinning (no kernel-level isolcpus/nohz_full/rcu_nocbs) is +expected physics (the periodic scheduler tick, device IRQs, shared-cache +contention that pinning alone can't remove), not a defect. No major +benchmarking harness (JMH, criterion.rs, Go's own `benchstat`) gates on a +fixed dispersion threshold either — all three gate on effect size against +the estimate's own uncertainty, same as `Significant`. + +`Diff.HighVariance` (CoV above `CoVCeiling`, default 0.25) is advisory only: +it flags a run worth a closer look (a noisy neighbor, a throttling event) but +never overrides `Significant`. The 0.25 default sits above the 4-8% CoV +measured as normal on a dedicated, pinned, bare-metal host with no +kernel-level isolation, and well below the ~40%+ territory a genuinely +pathological run would show. See the research doc linked above for the full +sweep (JMH/criterion.rs/benchstat comparison, isolcpus vs. taskset, the +practitioner consensus that isolcpus-level isolation is real-time/HFT-grade +overkill for this class of measurement) and for why the harness deliberately +does *not* add cpuset/IRQ-affinity isolation beyond `run.sh`'s taskset pin. + +## Active-benchmarking diagnostics: Nvcsw/Nivcsw + +`Series.NvcswDelta`/`NivcswDelta` (`gasbench.go`) are `getrusage(2)` +voluntary/involuntary context-switch counts over the timed window +(`RUSAGE_SELF` — process-wide, not thread-scoped; Go's `syscall` package +exposes no portable `RUSAGE_THREAD`). A nonzero `NivcswDelta` is direct, +measured evidence the kernel scheduler preempted a thread in this process +mid-window — Brendan Gregg's "active benchmarking" check, automated instead +of a one-off manual `perf stat`/`/proc/interrupts` session. It does not cover +every noise source: a high CoV with `NivcswDelta` near zero points at a +non-scheduler tail instead (the journal-growth reallocation above is the +known example). Near-zero `NvcswDelta` confirms the loop stayed on-CPU +without blocking, as expected for a pure-compute loop. + +`Diff` surfaces both as `BaselineNvcsw`/`TargetNvcsw` and +`BaselineNivcsw`/`TargetNivcsw`; `Run` surfaces the worse (max) of each pair +as `Nvcsw`/`Nivcsw`. A zero in either reflects an undisturbed window *or* a +failed `getrusage` call — check `Series.Warnings` (surfaced per-opcode via +`b.Logf` in `bench_test.go`) to tell the two apart. + +## Running it + +`run.sh` pins one measurement thread to one core (`GASBENCH_CORE`, default +3) and checks (but cannot enforce) turbo/governor/isolation settings — see +its header comment for the operator checklist. It runs: + +``` +go test ./tools/gasbench/ -run '^$' -bench '^BenchmarkOpcodes$' \ + -benchtime=1x -count=10 -benchmem +``` + +`-benchtime=1x` matters: `BenchmarkOpcodes`'s inner timing loop is +`Measure`'s, not `b.N`'s, so `b.N` must stay at 1. `-count=K` reruns the +benchmark K times **within the same OS process**, not as K independent +process forks — it gives `benchstat`-style cross-run variance, not +process-level independence. True process-level independence would need K +separate `go test` invocations. See the research doc for why this matters +less than it sounds: JMH's fork/warmup/measurement-iteration model is the +gold standard, but this MVP's within-process reruns already surfaced +reproducible per-opcode deltas (see the research doc's empirical run). + +Env vars: `GASBENCH_WARMUP`, `GASBENCH_ITERS` (override `DefaultConfig`), +`GASBENCH_SIGMA_K` (default 3), `GASBENCH_COV_CEILING` (default 0.25), +`GASBENCH_COUNT` (default 10), `GASBENCH_OUT_CSV`, `GASBENCH_OUT_NDJSON`. + +## Output schema + +One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON: + +| Field | Meaning | +|---|---| +| `input_id` | opcode id, e.g. `ADD` | +| `class` | opcode family, e.g. `arithmetic` | +| `reps` | opcode executions the delta represents | +| `gas_used` | whole-program gas delta (target - baseline); per-op = `gas_used/reps` | +| `exec_time_ns` | whole-program time delta, ns; per-op = `exec_time_ns/reps` | +| `status` | `ok` if `significant`, else `insignificant` — never gated on CoV | +| `iterations` | timed iterations behind each series | +| `cov` | worse (max) of the baseline/target series CoV — advisory only | +| `significant` | the acceptance gate | +| `high_variance` | advisory: CoV exceeded `CoVCeiling` | +| `nvcsw` / `nivcsw` | worse (max) of the baseline/target voluntary/involuntary context-switch counts | + +A per-`Case` self-check (`bench_test.go`) also verifies the measured gas +delta matches the definitional expected delta from `BuildCaseWith`'s algebra +— a correctness check on the harness construction itself, not on opcode +timing. All 22 `Specs` entries were hand-verified against the fork's +jump-table `minStack`/`maxStack`; verify a future addition the same way. + +## Scope + +Cleanly-benchmarkable-as-a-scalar opcodes only (arithmetic/bitwise/ +comparison/stack/control). Deferred, not yet implemented: parametric-curve +opcodes (`KECCAK256`/`EXP`/copy/memory/`LOG` by size), the state-touching +context-dependent matrix (`SLOAD`/`SSTORE` by warm/cold, `CALL` family, +`CREATE`), real-block replay (F1), and Sei's custom precompiles (0x1001+, +off-model — cosmos-module time, not EVM). diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index ada12fbe5d..b583f32a5d 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -7,9 +7,8 @@ import ( ) // BenchmarkOpcodes drives one differential measurement per scalar opcode -// Case. Run with -benchtime=1x (the inner loop is Measure's, not b.N) and -// -count=K to get K independent process runs for benchstat cross-run -// variance. +// Case. See README.md "Running it" for the required -benchtime=1x flag and +// what -count=K actually gives you (not K independent process forks). func BenchmarkOpcodes(b *testing.B) { cases := BuildCases() def := DefaultConfig() @@ -20,13 +19,7 @@ func BenchmarkOpcodes(b *testing.B) { LockThread: def.LockThread, } sigmaK := envFloat("GASBENCH_SIGMA_K", 3) - // 0.25: an advisory health-check ceiling, not the acceptance gate (that's - // Significant). Set well above the several-percent CoV plain core-pinning - // sees in practice (measured 4-8% on a dedicated bare-metal host with no - // kernel-level isolation), so it only fires on something genuinely - // pathological -- a noisy neighbor, a throttling event -- not routine - // scheduler-tick/IRQ noise. See - // designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md. + // 0.25 default: see README.md "Acceptance gate" for the calibration. covCeiling := envFloat("GASBENCH_COV_CEILING", 0.25) var runs []Run @@ -53,19 +46,10 @@ func BenchmarkOpcodes(b *testing.B) { d := Subtract(c.OpcodeID, base, tgt, c.Reps, sigmaK, covCeiling) - // Self-check of the differential construction: the measured - // whole-program gas delta must equal the definitional expected - // delta, or the baseline/target pair is not isolating the - // opcode the way BuildCaseWith intends. This is a correctness - // check on the harness, not on opcode timing. - // - // Note: this check is insensitive to a mis-transcribed - // OpSpec.Arity that still yields a self-consistent (if wrong) - // ExpectedGasDelta -- it catches a wrong ConstGas, a wrong - // opcode, or unexpected dynamic/memory gas, not an arity error. - // All 22 Specs entries were hand-verified against the fork's - // jump-table minStack/maxStack; verify a future addition the - // same way. + // Self-check of the harness construction (see README.md "Output + // schema"), not of opcode timing. Insensitive to a + // mis-transcribed OpSpec.Arity that still yields a + // self-consistent ExpectedGasDelta -- see AGENTS.md. if d.GasDelta != c.ExpectedGasDelta { b.Errorf("%s: gas self-check failed: measured delta %d != expected %d (baseline/target pair does not isolate the opcode)", c.OpcodeID, d.GasDelta, c.ExpectedGasDelta) diff --git a/tools/gasbench/diff.go b/tools/gasbench/diff.go index 19d4b5ee0d..c2ed71042f 100644 --- a/tools/gasbench/diff.go +++ b/tools/gasbench/diff.go @@ -2,25 +2,10 @@ package gasbench import "math" -// Diff is the subtraction of a baseline series (opcode replaced by a -// JUMPDEST/no-op) from a target series (opcode present). Everything the two -// programs share -- loop overhead, setup, timer cost -- cancels in the mean; -// what remains is attributable to the opcode. -// -// The estimator here is difference-of-medians with the median's asymptotic -// standard error propagated in quadrature. Medians reject the additive-noise -// tail better than means. A min-difference with a bootstrap CI is the -// estimator to reach for if the central assumption proves too coarse; it is -// the honest follow-up, not the MVP. -// -// Acceptance is Significant, not CoV. A raw per-series CoV of several percent -// under plain core-pinning (no kernel-level isolcpus/nohz_full/rcu_nocbs) is -// expected physics -- the periodic scheduler tick, device IRQs, and -// shared-cache contention that pinning alone cannot remove -- not a defect. -// No major benchmarking harness (JMH, criterion.rs, Go's own benchstat) gates -// on a fixed dispersion threshold either; all three gate on effect size -// against the estimate's own uncertainty, which is what Significant computes. -// See designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md. +// Diff is the difference-of-medians subtraction of a baseline series from a +// target series, with the median's standard error propagated in quadrature +// as Uncertainty. See README.md for the estimator choice and the +// Significant-not-CoV acceptance-gate rationale. type Diff struct { OpcodeID string @@ -34,20 +19,17 @@ type Diff struct { SigmaK float64 // significance multiple applied Significant bool // |DeltaNs| > SigmaK*Uncertainty -- the acceptance gate (see emit.go's Status) - // HighVariance is an advisory-only flag: true when either series' CoV - // exceeds CoVCeiling. It does not gate Status; it exists to catch a - // genuinely pathological run (a noisy neighbor, a throttling event) -- - // distinct from the routine several-percent CoV a pinned-but-not-isolated - // core sees, which HighVariance's default ceiling is set well above. + // HighVariance is advisory only (CoV exceeded CoVCeiling); it never gates + // Significant. See README.md "Acceptance gate". HighVariance bool CoVCeiling float64 - // BaselineNivcsw/TargetNivcsw are involuntary-context-switch counts - // observed during each series' timed window (process-wide; see - // Series.NivcswDelta). Nonzero values are direct evidence the kernel - // preempted the measurement thread mid-window -- the mechanism behind - // HighVariance and behind ordinary CoV, made observable rather than - // merely inferred. + // BaselineNvcsw/TargetNvcsw and BaselineNivcsw/TargetNivcsw are each + // series' voluntary/involuntary context-switch counts (see + // Series.NvcswDelta/NivcswDelta). See README.md "Active-benchmarking + // diagnostics". + BaselineNvcsw int64 + TargetNvcsw int64 BaselineNivcsw int64 TargetNivcsw int64 @@ -77,6 +59,8 @@ func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covCei TargetCoV: ts.CoV, SigmaK: sigmaK, CoVCeiling: covCeiling, + BaselineNvcsw: baseline.NvcswDelta, + TargetNvcsw: target.NvcswDelta, BaselineNivcsw: baseline.NivcswDelta, TargetNivcsw: target.NivcswDelta, Reps: reps, diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index 63b8c239c5..c57c84a842 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -9,32 +9,24 @@ import ( ) // Run is one emitted measurement row: the per-opcode differential result, -// ready for a gas-vs-time correlation. This is the record the design's -// acceptance criteria describe (per-input gas_used and exec_time_ns, -// supporting both a point lookup and a cross-input regression) -- for this -// harness "input" is the opcode, and the values that answer "does gas track -// time" are the differential (target-minus-baseline) gas and time, not -// either series' raw median. +// ready for a gas-vs-time correlation. See README.md "Output schema". type Run struct { InputID string `json:"input_id"` // opcode id, e.g. "ADD" Class string `json:"class"` // opcode family, e.g. "arithmetic" Reps int `json:"reps"` // opcode executions the delta represents GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps ExecTimeNs float64 `json:"exec_time_ns"` // whole-program time delta, ns (target median - baseline median); per-op = ExecTimeNs/Reps - Status string `json:"status"` // ok if the delta is statistically significant; insignificant otherwise. NOT gated on CoV -- see HighVariance. + Status string `json:"status"` // ok if Significant, else insignificant -- never gated on CoV Iterations int `json:"iterations"` // timed iterations behind each series - CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV -- informational, not the acceptance gate + CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV -- advisory only Significant bool `json:"significant"` // |delta| exceeds SigmaK * propagated uncertainty -- the acceptance gate - HighVariance bool `json:"high_variance"` // advisory: CoV exceeded the health-check ceiling; worth a look, does not invalidate Status=ok - Nivcsw int64 `json:"nivcsw"` // worse (max) of the baseline/target involuntary-context-switch count; 0 means undisturbed by scheduler preemption + HighVariance bool `json:"high_variance"` // advisory: CoV exceeded the health-check ceiling; does not invalidate Status=ok + Nvcsw int64 `json:"nvcsw"` // worse (max) of the baseline/target voluntary-context-switch count + Nivcsw int64 `json:"nivcsw"` // worse (max) of the baseline/target involuntary-context-switch count; see README.md "Active-benchmarking diagnostics" for interpreting a zero here } -// Status values. -// -// Acceptance is Significant, not CoV: a raw per-series CoV of several -// percent under plain core-pinning is expected physics, not a defect (see -// designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md). -// CoV rides as the advisory HighVariance flag instead of gating Status. +// Status values. See README.md "Acceptance gate" for why Status is +// Significant-driven, not CoV-driven. const ( StatusOK = "ok" // Significant is true -- the delta is trustworthy regardless of routine CoV StatusInsignificant = "insignificant" // Significant is false -- the delta does not clear its own measurement uncertainty; needs more Iterations or a coarser SigmaK, not a quieter host @@ -54,6 +46,10 @@ func NewRun(c Case, d Diff, iterations int) Run { if d.TargetCoV > cov { cov = d.TargetCoV } + nvcsw := d.BaselineNvcsw + if d.TargetNvcsw > nvcsw { + nvcsw = d.TargetNvcsw + } nivcsw := d.BaselineNivcsw if d.TargetNivcsw > nivcsw { nivcsw = d.TargetNivcsw @@ -73,11 +69,12 @@ func NewRun(c Case, d Diff, iterations int) Run { CoV: cov, Significant: d.Significant, HighVariance: d.HighVariance, + Nvcsw: nvcsw, Nivcsw: nivcsw, } } -var csvHeader = []string{"input_id", "class", "reps", "gas_used", "exec_time_ns", "status", "iterations", "cov", "significant", "high_variance", "nivcsw"} +var csvHeader = []string{"input_id", "class", "reps", "gas_used", "exec_time_ns", "status", "iterations", "cov", "significant", "high_variance", "nvcsw", "nivcsw"} // WriteCSV writes a header plus one row per run. func WriteCSV(w io.Writer, runs []Run) error { @@ -97,6 +94,7 @@ func WriteCSV(w io.Writer, runs []Run) error { strconv.FormatFloat(r.CoV, 'g', 6, 64), strconv.FormatBool(r.Significant), strconv.FormatBool(r.HighVariance), + strconv.FormatInt(r.Nvcsw, 10), strconv.FormatInt(r.Nivcsw, 10), } if err := cw.Write(rec); err != nil { diff --git a/tools/gasbench/emit_test.go b/tools/gasbench/emit_test.go new file mode 100644 index 0000000000..e8c3ebbca5 --- /dev/null +++ b/tools/gasbench/emit_test.go @@ -0,0 +1,30 @@ +package gasbench + +import "testing" + +// TestNewRunStatusIsSignificantOnly pins that Status is a pure function of +// Significant: HighVariance (CoV) must never flip it. See README.md +// "Acceptance gate". +func TestNewRunStatusIsSignificantOnly(t *testing.T) { + c := Case{OpcodeID: "ADD", Class: ClassArithmetic} + + cases := []struct { + significant bool + highVariance bool + want string + }{ + {significant: true, highVariance: false, want: StatusOK}, + {significant: true, highVariance: true, want: StatusOK}, + {significant: false, highVariance: false, want: StatusInsignificant}, + {significant: false, highVariance: true, want: StatusInsignificant}, + } + + for _, tc := range cases { + d := Diff{Significant: tc.significant, HighVariance: tc.highVariance} + got := NewRun(c, d, 0).Status + if got != tc.want { + t.Errorf("Significant=%v HighVariance=%v: Status = %q, want %q", + tc.significant, tc.highVariance, got, tc.want) + } + } +} diff --git a/tools/gasbench/gasbench.go b/tools/gasbench/gasbench.go index edff0316a9..f84dc4854b 100644 --- a/tools/gasbench/gasbench.go +++ b/tools/gasbench/gasbench.go @@ -1,9 +1,6 @@ // Package gasbench measures per-opcode EVM execution time and correlates it -// with gas cost via differential microbenchmarking. -// -// The harness is tracer-free by construction: nothing here attaches an EVM -// tracer, so the interpreter runs its hot path unobserved. Timing uses the -// monotonic clock (time.Now/time.Since) around a caller-supplied RunOnce. +// with gas cost via differential microbenchmarking. See README.md for the +// full methodology and rationale. package gasbench import ( @@ -15,28 +12,19 @@ import ( ) // RunOnce executes a pre-loaded EVM program to completion through the -// tracer-free interpreter and reports the gas it consumed. -// -// A RunOnce closes over its program (see Program.Run in program.go) so the -// timing loop never rebuilds interpreter/StateDB state per call. Rebuilding -// per call would swamp a nanosecond-scale opcode signal with allocation -// noise; the differential construction only holds if baseline and target pay -// identical, amortized-out setup cost. +// tracer-free interpreter and reports the gas it consumed. It closes over +// its program (see Program.Run) so the timing loop never rebuilds +// interpreter/StateDB state per call. // -// Contract the EVM-wiring side must satisfy: +// Contract: // - Deterministic: identical program yields identical gasUsed and -// equivalent work every call; no state carried across calls that would -// drift the timing (Program.Run resets transient storage/access list -// per entry, which is safe only for pure-compute, non-state-touching -// programs). +// equivalent work every call. // - Self-contained and tracer-free: no I/O, logging, or tracer on the hot -// path; the only thing measured is interpreter execution. -// - Sufficient work: the program runs the opcode-under-test enough times -// that total runtime is well above timer resolution (target >= ~10us), -// so per-call time.Now overhead and clock granularity do not dominate. -// - Allocation-light: GC is disabled during the window, so allocations -// across warmup+Iterations must fit in RAM. Keep the program lean or -// lower Iterations. +// path. +// - Sufficient work: total runtime well above timer resolution +// (target >= ~10us). +// - Allocation-light: GC is disabled during the window (see Measure), so +// allocations across warmup+Iterations must fit in RAM. type RunOnce func() (gasUsed uint64, err error) // sink defeats dead-code elimination of the gas result. A single measurement @@ -64,24 +52,16 @@ type Series struct { Samples []time.Duration Warnings []string - // NvcswDelta/NivcswDelta are process-wide (RUSAGE_SELF, not thread-scoped - // -- Go exposes no portable RUSAGE_THREAD) voluntary/involuntary - // context-switch counts observed during the timed window (Warmup - // excluded). This is the cheapest "active benchmarking" check available - // (Gregg): a nonzero NivcswDelta on a nominally pinned core is direct - // confirmation the kernel scheduler preempted the measurement thread - // mid-window -- the same tick/IRQ/neighbor mechanism a CoV reading only - // infers indirectly. See - // designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md. + // NvcswDelta/NivcswDelta are process-wide (RUSAGE_SELF) voluntary/ + // involuntary context-switch counts over the timed window (Warmup + // excluded). See README.md "Active-benchmarking diagnostics". NvcswDelta int64 NivcswDelta int64 } // rusageSnapshot reads the process's current voluntary/involuntary -// context-switch counters. Process-wide rather than thread-scoped, so on a -// process with other active goroutines the delta can include their -// scheduling activity too; for this harness's single-goroutine measurement -// loop that's a minor overcount, not a different mechanism. +// context-switch counters (process-wide, not thread-scoped: Go's syscall +// package exposes no portable RUSAGE_THREAD). func rusageSnapshot() (nvcsw, nivcsw int64, err error) { var ru syscall.Rusage if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { diff --git a/tools/gasbench/program.go b/tools/gasbench/program.go index ce1c58a46a..31d9ec18c5 100644 --- a/tools/gasbench/program.go +++ b/tools/gasbench/program.go @@ -15,15 +15,9 @@ import ( var contractAddr = common.BytesToAddress([]byte("contract")) // newRuntimeConfig builds a fresh, tracer-free interpreter config against a -// fresh in-memory StateDB. -// -// This does NOT use runtime.Execute. Execute discards leftOverGas (it only -// surfaces gas via a tracer's OnTxEnd hook, and we run tracer-free - -// EVMConfig.Tracer == nil, the debug=false path), so it cannot return gas. -// runtime.Call is the identical fresh-EVM, tracer-free code path but returns -// leftOverGas directly. We replicate Execute's fresh-state setup -// (state.New + CreateAccount + SetCode) so the "fresh in-memory StateDB, -// stateless" property this MVP relies on is preserved. +// fresh in-memory StateDB. Uses runtime.Call, not runtime.Execute: Execute +// discards leftOverGas (see Program.Run). Replicates Execute's fresh-state +// setup (state.New + CreateAccount + SetCode). func newRuntimeConfig(code []byte) (*runtime.Config, error) { cfg := new(runtime.Config) runtime.SetDefaults(cfg) // Shanghai+Cancun active, GasLimit=MaxUint64, no tracer @@ -40,23 +34,8 @@ func newRuntimeConfig(code []byte) (*runtime.Config, error) { } // Program holds a warmed EVM environment so a timing loop measures only the -// hot Call, not StateDB construction (which would swamp a nanosecond-scale -// opcode signal). Build one Program per bytecode input; call Run in the -// timing loop. -// -// Reusing a single StateDB across many Run calls is safe for these -// pure-compute programs, but not because "no state change occurs": every -// call's evm.Call takes a Snapshot() that is never reverted or Finalise-d -// (Finalise is what resets the journal; this harness never calls it), so the -// journal's revision/touch-change list grows roughly two entries per call -// across the life of a Program. That growth is harmless here because (a) gas -// accounting is unaffected -- it comes only from interpreter opcodes -- and -// (b) the growth is symmetric between baseline and target (both take exactly -// one Snapshot + one zero-value Transfer per call) and lands as tail latency -// from periodic slice reallocation, not a shift in the median the -// difference-of-medians estimator reads. Do NOT assume this still holds if -// Iterations grows by an order of magnitude, or if a future Case touches -// persistent state (SSTORE/LOG/CREATE) -- re-check the journal-growth impact +// hot Call, not StateDB construction. Build one Program per bytecode input; +// call Run in the timing loop. See README.md "Program reuse across calls" // before reusing a Program across a much longer or state-touching series. type Program struct { cfg *runtime.Config @@ -77,22 +56,15 @@ func NewProgram(code []byte) (*Program, error) { return p, nil } -// Run executes the loaded code once, returning EVM gas consumed. This is the -// hot function for the timing loop: it does no per-call state allocation. -// -// gasUsed is cfg.GasLimit - leftOverGas: total EVM gas the interpreter -// charged. There is deliberately NO Cosmos/Sei gas meter in this path - -// runtime.Call builds a bare go-ethereum EVM with no ante handler and no -// sei-cosmos GasMeter. Isolating the EVM interpreter from the Cosmos layer is -// the entire point of the stateless-compute MVP. +// Run executes the loaded code once through a bare go-ethereum EVM (no +// Cosmos ante handler or GasMeter -- see README.md "Gas isolation from the +// Cosmos layer") and returns EVM gas consumed (cfg.GasLimit - leftOverGas). // -// err is nil on a clean STOP/RETURN. On REVERT err is vm.ErrExecutionReverted -// (gasUsed is partial - unused gas is refunded on revert). On out-of-gas err -// is vm.ErrOutOfGas (gasUsed == GasLimit). On a bad/undefined opcode err is -// vm.ErrInvalidOpCode. The differential programs built in programs.go -// terminate cleanly (STOP, balanced stack), so a non-nil err means the run -// is INVALID - the caller must discard the sample, never treat it as a -// measurement. Matches the gasbench.RunOnce contract (gasbench.go). +// err is nil on a clean STOP/RETURN, vm.ErrExecutionReverted on REVERT +// (gasUsed partial), vm.ErrOutOfGas on out-of-gas (gasUsed == GasLimit), or +// vm.ErrInvalidOpCode on a bad opcode. Every Case built by programs.go +// terminates cleanly, so a non-nil err means the run is invalid -- see the +// RunOnce contract in gasbench.go. func (p *Program) Run() (gasUsed uint64, err error) { _, leftOverGas, callErr := runtime.Call(contractAddr, nil, p.cfg) return p.cfg.GasLimit - leftOverGas, callErr diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index f0852dbd74..467328520f 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -7,11 +7,8 @@ import ( "github.com/holiman/uint256" ) -// DefaultReps is the number of unrolled body units per program. Straight-line -// (no loop counter) keeps the interpreter dispatch loop identical between -// baseline and target; the count is fixed and equal for both, so loop-control -// cost cannot leak into the differential. 1000 gives a strong signal while -// staying well under any code/gas limit. +// DefaultReps is the number of unrolled (straight-line, no loop counter) +// body units per program, fixed and equal for baseline and target. const DefaultReps = 1000 // Class groups opcodes by arithmetic character. @@ -32,24 +29,15 @@ type OpSpec struct { Arity int // stack items consumed by the op (operands) ConstGas uint64 // definitional constant gas from the fork jump table // DataDependent is true when execution TIME varies with operand - // magnitude (uint256 limb-count loops) even though gas is constant. - // Treat these as curves, not points, if the harness ever sweeps operands - // (deferred - see the design's Non-goals). + // magnitude even though gas is constant. See README.md "Scope". DataDependent bool } -// Specs is the benchmarkable scalar-opcode set. Every entry is CONSTANT-gas -// in this fork (verified against core/vm/jump_table.go + core/vm/eips.go); -// none has dynamic gas, none touches memory/state. EXP is deliberately -// omitted: its gas is parametric (gasExpFrontier over exponent byte length), -// so it does not belong in a constant-gas differential. -// -// No scalar opcode here is repriced on Sei: production always builds the EVM -// with a stock vm.Config{} (x/evm/keeper/evm.go, msg_server.go, ante/fee.go), -// no custom jump table, no opcode gas overrides. The only Sei gas param is -// SeiSstoreSetGasEIP2200 (x/evm/types/params.go), a storage opcode and out of -// scope here. If a scalar opcode is ever repriced, its gas must come from the -// live x/evm params, not this table's geth constant. +// Specs is the benchmarkable scalar-opcode set: every entry is +// constant-gas in this fork (verified against core/vm/jump_table.go + +// core/vm/eips.go), none touches memory/state. EXP is omitted: its gas is +// parametric (gasExpFrontier), not constant. See AGENTS.md for the +// hand-verification requirement when adding an entry. var Specs = []OpSpec{ // arithmetic {"ADD", vm.ADD, ClassArithmetic, 2, vm.GasFastestStep, false}, @@ -79,17 +67,13 @@ var Specs = []OpSpec{ {"SWAP1", vm.SWAP1, ClassStack, 0, vm.GasFastestStep, false}, } -// Case is a differential bytecode pair for one opcode, ready for measurement. -// Baseline and target are identical except that target executes the opcode; -// both run the same fixed unit count and both terminate cleanly on STOP with -// a balanced stack. +// Case is a differential bytecode pair for one opcode, ready for +// measurement. See README.md "Differential construction". // -// ExpectedGasDelta is the definitional, whole-program gas the opcode adds -// (per-unit delta * Reps) - NOT necessarily Reps*ConstGas, because a net-0, -// cleanly-looping body needs stack rebalancing (see BuildPairWith). It is the -// expected value; the measured Diff.GasDelta from a live run is the ground -// truth, and the two should be cross-checked (see bench_test.go) as a -// self-check of the differential construction itself. +// ExpectedGasDelta is the definitional whole-program gas delta from +// BuildCaseWith's algebra (not necessarily Reps*ConstGas -- stack +// rebalancing adds its own cost); bench_test.go cross-checks it against the +// measured Diff.GasDelta as a self-check of the construction itself. type Case struct { OpcodeID string Class Class @@ -100,20 +84,13 @@ type Case struct { ExpectedGasDelta uint64 } -// seed256 is the fixed operand pushed once as the working value. Full-width -// (2^256-1) so the magnitude-dependent ops (MUL/DIV/MOD/ADDMOD/MULMOD) -// exercise their full uint256 limb path rather than a small-number fast -// case. Exposed via BuildCaseWith for operand sweeps. +// seed256 is the fixed working-value operand: full-width (2^256-1) so +// magnitude-dependent ops exercise their full uint256 limb path. var seed256 = new(uint256.Int).Not(uint256.NewInt(0)) -// seedShift is a fixed, in-range (<256) shift amount for SHL/SHR/SAR. A -// shift amount >= 256 takes go-ethereum's value.Clear() early-out (the -// degenerate, cheapest case: the result is zero without touching the -// value's limbs), so reusing seed256 as BOTH operands -- as the generic -// same-value construction below does for every other opcode -- measures -// that early-out, not representative shift work. Kept separate from the -// value operand so the shift ops still exercise the interpreter's limb-shift -// path on a full-width value. +// seedShift is a fixed, in-range (<256) shift amount for SHL/SHR/SAR -- +// see README.md "Differential construction" for why this must differ from +// seed256. var seedShift = uint256.NewInt(4) // BuildCases builds every spec's case at DefaultReps. @@ -125,41 +102,10 @@ func BuildCases() []Case { return out } -// BuildCaseWith constructs the differential case for one opcode. -// -// Construction (the balanced-DUP/POP differential, the fork's -// benchmarkNonModifyingCode technique adapted to a bounded, clean-terminating -// straight line). For an op consuming n operands and producing 1 result, the -// net-0 repeating unit is: -// -// target = DUP1 x n , OP , POP // dup n copies of top, run op, drop result -// baseline = DUP1 x n , POP x n // dup n copies of top, drop them -// -// The n DUP1s are identical on both sides and cancel; one POP cancels the -// target's trailing POP. What remains -- the whole differential -- is the op -// vs (n-1) extra POPs, so the per-unit gas delta = ConstGas(op) - (n-1)*GasQuickStep. -// - n=1 (NOT, ISZERO): delta = ConstGas(op) (op isolated exactly) -// - n=2 (most ops): delta = ConstGas(op) - 2 -// - n=3 (ADDMOD/MULMOD): delta = ConstGas(op) - 4 -// -// Stack ops are special-cased (not "n operands -> 1 result"): -// - DUP1 (1->2): target = DUP1 POP ; baseline = PUSH0 POP -> delta = 3-2 = 1 -// - SWAP1 (2->2): target = SWAP1 ; baseline = JUMPDEST -> delta = 3-1 = 2 -// -// SHL/SHR/SAR are also special-cased: they need two DISTINCT operands (see -// seedShift), not "n copies of the same value". The prologue pushes -// (seed, seedShift); DUP2 reaches two positions down without disturbing the -// other operand, so the SAME (value, shift) pair is reused net-0 across -// every unit: -// -// target = DUP2, DUP2, OP , POP // dup shift then value, run op, drop result -// baseline = DUP2, DUP2, POP, POP // dup shift then value, drop both -// -// This yields the same arity=2 shape as the general case: -// delta = ConstGas(op) - GasQuickStep (one extra POP in baseline vs target). -// -// Every case's prologue seeds its own stack, and every unit is net-0 so stack -// depth never grows (no 1024-depth risk) and never underflows. +// BuildCaseWith constructs the differential case for one opcode. See +// README.md "Differential construction" for the balanced-DUP/POP algebra; +// per-branch delta formulas are inlined below next to the code they justify. +// Every unit is net-0 so stack depth never grows or underflows. func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { base := program.New() tgt := program.New() diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh index 1841dce800..babcf8a1a2 100755 --- a/tools/gasbench/run.sh +++ b/tools/gasbench/run.sh @@ -30,12 +30,7 @@ if [ -r "$gov" ]; then fi export GOMAXPROCS=1 -# GASBENCH_ITERS / GASBENCH_WARMUP are left unset here by default: -# gasbench.DefaultConfig() in Go is the single source of truth for those two. -# GASBENCH_COV_CEILING (an advisory health-check, not the acceptance gate -- -# see emit.go/diff.go) / GASBENCH_SIGMA_K default in bench_test.go instead (no -# Config field for them). Set any of these in the environment only to -# override for this run. +# See README.md "Running it" for the full env-var list and defaults. export GASBENCH_OUT_CSV="${GASBENCH_OUT_CSV:-gasbench.csv}" export GASBENCH_OUT_NDJSON="${GASBENCH_OUT_NDJSON:-gasbench.ndjson}" diff --git a/tools/gasbench/stats.go b/tools/gasbench/stats.go index ccc291b92d..218b5894db 100644 --- a/tools/gasbench/stats.go +++ b/tools/gasbench/stats.go @@ -15,7 +15,7 @@ type Stats struct { Median float64 P99 float64 Stddev float64 // sample stddev (n-1) - CoV float64 // Stddev/Mean; the acceptance gate ("noise floor") + CoV float64 // Stddev/Mean; advisory only -- see Diff.Significant, README.md "Acceptance gate" SEMean float64 // standard error of the mean SEMedian float64 // asymptotic standard error of the median } From eb6431f9d9b11ab45ab81f602133fc4501e5ee65 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 17:15:42 -0700 Subject: [PATCH 06/19] fix(tools/gasbench): correct the doc-migration's own factual error xreview R2's assigned dissenter (systems-engineer) found that README.md's "load-bearing invariant" section, written this round, claimed gas and time come from two separate tracer-free runs. They don't: gasbench.go's Measure times the exact same run() call whose return value becomes GasUsed. This harness only measures whole-program gas, which needs no tracer at all, so there's no separate gas pass to speak of -- the real invariant is "never attach a tracer to the timed call." Fixed in both README.md and AGENTS.md (which restated the same error), and added a note that a future per-opcode gas breakdown (which would need a tracer) must come from a separate, untimed call. Also, from the same round's secondary findings: - emit_test.go's table test asserted Status is independent of HighVariance but never varied BaselineCoV/TargetCoV, so a regression gating Status on raw CoV directly would have passed unnoticed. Added two cases with CoV=0.9 to actually exercise that path. - Added a same-value doc pointer to Nvcsw (Nivcsw already had one) for the getrusage-failure-zero ambiguity. - Added inline pointers at the SHL/SHR/SAR and default BuildCaseWith branches back to README.md, since the correctness reasoning for DUP2-vs-DUP1 no longer lives next to the switch. - Added the new files to AGENTS.md's Files table. And from prose-steward's independent pass over the same two new docs: - README's invariant header now matches AGENTS's framing exactly (both fixes above happened to close prose-steward's top finding too: the two docs stated the load-bearing invariant at different scopes). - Reconciled AGENTS's "terminate cleanly (STOP...)" with README's general STOP/RETURN error contract -- every Case built here uses STOP by construction, but the general Program.Run contract also accepts RETURN. - Reconciled the new-opcode-spec verification recipe: AGENTS is now the one authoritative checklist (bridges Arity to jump_table.go's minStack/maxStack); README points to it instead of restating a partial version. - Expanded CoV on first use, fixed a forward reference to the getrusage section, dropped a hardcoded "22 Specs entries" count that would rot on the next addition. Local branch only, not pushed. --- tools/gasbench/AGENTS.md | 33 ++++++++++++++++++++++----------- tools/gasbench/README.md | 37 +++++++++++++++++++++---------------- tools/gasbench/emit.go | 2 +- tools/gasbench/emit_test.go | 29 ++++++++++++++++++++--------- tools/gasbench/programs.go | 6 ++++++ 5 files changed, 70 insertions(+), 37 deletions(-) diff --git a/tools/gasbench/AGENTS.md b/tools/gasbench/AGENTS.md index 667f79807b..98c140566f 100644 --- a/tools/gasbench/AGENTS.md +++ b/tools/gasbench/AGENTS.md @@ -8,19 +8,27 @@ file is the short orientation. ## Rules for changes here - **Never attach a tracer to a timed run.** `debug=true` in the interpreter - dilates per-step time — gas and time are always measured in separate, - tracer-free runs. See README.md. + dilates per-step time. Whole-program gas needs no tracer, so it's read off + the same tracer-free call that's timed — there is no separate gas-only + pass. A future per-opcode gas breakdown *would* need a tracer, and that + breakdown must come from a separate, untimed call. See README.md. - **Correlate against EVM gas, never the Cosmos gas meter.** `Program.Run` deliberately has no ante handler / `GasMeter` in its path. -- **A new `Case` must terminate cleanly** (STOP, balanced stack, net-zero - gas per unit) — `NewProgram` rejects anything that doesn't run clean once, - and `bench_test.go`'s self-check will fail loudly if the algebra is wrong. -- **New opcode specs:** hand-verify `Arity`/`ConstGas` against the fork's - `core/vm/jump_table.go` + `core/vm/eips.go`; the self-check catches a wrong - `ConstGas` but not a self-consistent wrong `Arity`. `ConstGas` must come - from the geth constant, not a Sei override — production never reprices a - scalar opcode (stock `vm.Config{}`, no custom jump table); the only Sei gas - param is `SeiSstoreSetGasEIP2200`, a storage opcode and out of scope here. +- **A new `Case` must terminate cleanly** — every `Case` `BuildCaseWith` + builds ends in STOP with a balanced, net-zero-gas stack by construction; + `NewProgram` rejects anything that doesn't run clean once, and + `bench_test.go`'s self-check will fail loudly if the algebra is wrong. (The + general `Program.Run` contract also accepts RETURN — see README.md "Error + contract" — but every `Case` built here uses STOP.) +- **New opcode specs:** hand-verify `Arity` (against the fork's + `core/vm/jump_table.go` `minStack`/`maxStack`) and `ConstGas` (against + `core/vm/jump_table.go` + `core/vm/eips.go`); the self-check catches a + wrong `ConstGas` but not a self-consistent wrong `Arity` — this is the + authoritative verification checklist, README.md's mention of it points + back here. `ConstGas` must come from the geth constant, not a Sei + override — production never reprices a scalar opcode (stock `vm.Config{}`, + no custom jump table); the only Sei gas param is `SeiSstoreSetGasEIP2200`, + a storage opcode and out of scope here. - Keep code comments lean (what, not why); put methodology/rationale in README.md instead of growing doc comments. @@ -43,5 +51,8 @@ Env vars, output schema, and the `-count` semantics caveat: see README.md | `program.go` | `Program`: warmed tracer-free EVM environment, one bytecode input | | `programs.go` | `OpSpec`/`Specs`/`Case`/`BuildCaseWith`: the differential bytecode construction | | `emit.go` | `Run`, CSV/NDJSON output | +| `emit_test.go` | pins `Run.Status` as a pure function of `Diff.Significant` | | `bench_test.go` | `BenchmarkOpcodes`: wires the above into `go test -bench` | | `run.sh` | pinned-core runner + operator checklist for turbo/governor/isolation | +| `README.md` | full rationale: construction, acceptance gate, diagnostics | +| `AGENTS.md` | this file | diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index 1e30ee9175..0d07c1cbef 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -9,19 +9,22 @@ the `bdchatham-designs` repo. Noise-floor calibration behind the acceptance gate below: `designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md` in the same repo. -## The load-bearing invariant: never measure gas and time in the same run +## The load-bearing invariant: never attach a tracer to a timed call Attaching a `tracing.Hooks` tracer to sum per-opcode gas sets `debug=true` in -the interpreter loop, which dilates per-step execution time. So: - -- **Time** comes from a tracer-free run (`vm.Config{}`, nil `Tracer` — - production's own hot path). -- **Gas** is deterministic and comes from a separate, tracer-free run too - (`GasLimit - leftOverGas` via `runtime.Call`, not `runtime.Execute`, which - discards `leftOverGas`). - -Both series in a `Case` (baseline and target) are measured this way, so -gas and time are never read off the same call. +the interpreter loop, which dilates per-step execution time. This harness +only measures whole-program gas (not a per-opcode breakdown), and +whole-program gas needs no tracer at all: `Program.Run` +(`GasLimit - leftOverGas` via `runtime.Call`, not `runtime.Execute`, which +discards `leftOverGas`) returns it directly. So the *same* tracer-free call +that's timed also yields its gas — `Measure` (`gasbench.go`) reads `gasUsed` +from the identical `run()` invocation that `time.Since` wraps. There is no +separate gas-only pass in this harness. + +**If a future phase adds a per-opcode gas breakdown**, that requires +attaching a tracer, and the timed run must stay tracer-free — that +breakdown would have to come from a *separate* tracer-on call whose timing +is never read, not from the timed call. ## Differential construction @@ -79,8 +82,8 @@ order-of-magnitude more iterations, or for a `Case` that touches persistent state (SSTORE/LOG/CREATE).** This periodic reallocation is also the one CoV-inflating noise source the -`getrusage` diagnostics below structurally cannot see (it's not a scheduler -preemption). +`getrusage`-based diagnostics ("Active-benchmarking diagnostics" below) +structurally cannot see (it's not a scheduler preemption). ## Error contract @@ -109,7 +112,8 @@ I-cache/D-cache/branch predictor, catch a broken program early), then `Diff.Significant` (`diff.go`) — `|DeltaNs| > SigmaK * Uncertainty`, where `Uncertainty` is the two series' median standard errors propagated in -quadrature — is the acceptance gate. A raw per-series CoV of several percent +quadrature — is the acceptance gate. A raw per-series coefficient of +variation (CoV, stddev/mean) of several percent under plain core-pinning (no kernel-level isolcpus/nohz_full/rcu_nocbs) is expected physics (the periodic scheduler tick, device IRQs, shared-cache contention that pinning alone can't remove), not a defect. No major @@ -194,8 +198,9 @@ One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON: A per-`Case` self-check (`bench_test.go`) also verifies the measured gas delta matches the definitional expected delta from `BuildCaseWith`'s algebra — a correctness check on the harness construction itself, not on opcode -timing. All 22 `Specs` entries were hand-verified against the fork's -jump-table `minStack`/`maxStack`; verify a future addition the same way. +timing. It catches a wrong `ConstGas` but not a self-consistent wrong +`Arity`; see `AGENTS.md` "New opcode specs" for the hand-verification +checklist a new `Specs` entry needs. ## Scope diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index c57c84a842..601ba13c7f 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -21,7 +21,7 @@ type Run struct { CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV -- advisory only Significant bool `json:"significant"` // |delta| exceeds SigmaK * propagated uncertainty -- the acceptance gate HighVariance bool `json:"high_variance"` // advisory: CoV exceeded the health-check ceiling; does not invalidate Status=ok - Nvcsw int64 `json:"nvcsw"` // worse (max) of the baseline/target voluntary-context-switch count + Nvcsw int64 `json:"nvcsw"` // worse (max) of the baseline/target voluntary-context-switch count; see README.md "Active-benchmarking diagnostics" for interpreting a zero here Nivcsw int64 `json:"nivcsw"` // worse (max) of the baseline/target involuntary-context-switch count; see README.md "Active-benchmarking diagnostics" for interpreting a zero here } diff --git a/tools/gasbench/emit_test.go b/tools/gasbench/emit_test.go index e8c3ebbca5..3125fbc113 100644 --- a/tools/gasbench/emit_test.go +++ b/tools/gasbench/emit_test.go @@ -3,28 +3,39 @@ package gasbench import "testing" // TestNewRunStatusIsSignificantOnly pins that Status is a pure function of -// Significant: HighVariance (CoV) must never flip it. See README.md -// "Acceptance gate". +// Significant: neither HighVariance nor the raw CoV values behind it may +// flip it. See README.md "Acceptance gate". func TestNewRunStatusIsSignificantOnly(t *testing.T) { c := Case{OpcodeID: "ADD", Class: ClassArithmetic} cases := []struct { significant bool highVariance bool + cov float64 // fed into BaselineCoV/TargetCoV directly want string }{ - {significant: true, highVariance: false, want: StatusOK}, - {significant: true, highVariance: true, want: StatusOK}, - {significant: false, highVariance: false, want: StatusInsignificant}, - {significant: false, highVariance: true, want: StatusInsignificant}, + {significant: true, highVariance: false, cov: 0, want: StatusOK}, + {significant: true, highVariance: true, cov: 0, want: StatusOK}, + {significant: false, highVariance: false, cov: 0, want: StatusInsignificant}, + {significant: false, highVariance: true, cov: 0, want: StatusInsignificant}, + // A regression that gated Status on CoV directly (rather than on + // Significant/HighVariance) would flip these two, even though the + // cases above never exercise a nonzero CoV. + {significant: true, highVariance: true, cov: 0.9, want: StatusOK}, + {significant: false, highVariance: true, cov: 0.9, want: StatusInsignificant}, } for _, tc := range cases { - d := Diff{Significant: tc.significant, HighVariance: tc.highVariance} + d := Diff{ + Significant: tc.significant, + HighVariance: tc.highVariance, + BaselineCoV: tc.cov, + TargetCoV: tc.cov, + } got := NewRun(c, d, 0).Status if got != tc.want { - t.Errorf("Significant=%v HighVariance=%v: Status = %q, want %q", - tc.significant, tc.highVariance, got, tc.want) + t.Errorf("Significant=%v HighVariance=%v CoV=%v: Status = %q, want %q", + tc.significant, tc.highVariance, tc.cov, got, tc.want) } } } diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index 467328520f..e7a2871966 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -131,6 +131,10 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { perUnitDelta = s.ConstGas - params.JumpdestGas // 3 - 1 case vm.SHL, vm.SHR, vm.SAR: + // DUP2 (not DUP1 x2, unlike the default branch below): these ops + // need two DISTINCT operands, not n copies of one value -- see + // README.md "Differential construction" (the value.Clear() early-out + // a same-value construction would accidentally measure). base.Push(seed).Push(seedShift) tgt.Push(seed).Push(seedShift) for i := 0; i < reps; i++ { @@ -140,6 +144,8 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { perUnitDelta = s.ConstGas - vm.GasQuickStep // 3 - 2, same shape as the n=2 default case default: + // General n-operand case: see README.md "Differential construction" + // for the (n-1)*GasQuickStep derivation this formula implements. base.Push(seed).Push(seed) tgt.Push(seed).Push(seed) for i := 0; i < reps; i++ { From 2d7b4a552ece72bad0b15b365e1a3200059f3bb0 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 17:26:07 -0700 Subject: [PATCH 07/19] fix(tools/gasbench): never report Significant from an unestimated Uncertainty xreview R3's assigned dissenter (systems-engineer) ran a from-scratch correctness audit (not just re-verifying prior fixes) and found a real bug in the acceptance gate itself: Summarize only computes Stddev/SEMean/SEMedian when a series has more than 1 sample, so at Iterations=1 (or any degenerate zero-variance sample) Uncertainty is exactly 0. Significant was `|DeltaNs| > SigmaK*0`, trivially true for any nonzero delta -- every opcode reported significant=true with a statistically meaningless uncertainty of zero. GASBENCH_ITERS is operator-settable with no documented floor, so this was reachable, not theoretical. Verified empirically: at Iterations=1 every one of the 22 opcodes (including MULMOD's 44us delta) used to report significant=true; Subtract now requires Uncertainty > 0, so all 22 correctly report insignificant instead. Added TestSubtractNeverSignificantWithZeroUncertainty pinning this. Also closes the remaining Round 3 findings, all RATIFY-with-advisory from idiomatic-reviewer, solidity-developer, and prose-steward: - gasbench.go: fixed the sink comment's wrong justification (it isn't GOMAXPROCS=1/thread-locking that makes the shared var race-free, it's that BenchmarkOpcodes's subtests run sequentially). - bench_test.go: removed a redundant c := c loop-variable copy (dead under Go 1.22+ per-iteration loop-var semantics, and b.Run here never runs in parallel). - programs.go: added a Class const-block group comment for symmetry with Status's; trimmed the SHL/SHR/SAR inline comment back to a README pointer instead of restating go-ethereum internals; added an explicit Arity<1 guard in BuildCaseWith's default branch (the (n-1)*GasQuickStep formula assumes n>=1; DUP1/SWAP1 are the only Arity-0 specs and are already special-cased above it). - emit.go: documented the CSV-vs-NDJSON CoV precision asymmetry (6-sig-fig display rounding vs full precision) as a deliberate, advisory-only choice rather than leaving it silent. - README.md: fixed SAR's early-out boundary (strictly >256, vs SHL/SHR's >=256); replaced a line-number anchor into x/evm/keeper/msg_server.go with a symbol reference so it can't rot; moved the CoV term's expansion to its actual first use. Local branch only, not pushed. --- tools/gasbench/README.md | 26 +++++++++++++---------- tools/gasbench/bench_test.go | 1 - tools/gasbench/diff.go | 6 +++++- tools/gasbench/diff_test.go | 40 ++++++++++++++++++++++++++++++++++++ tools/gasbench/emit.go | 4 ++++ tools/gasbench/gasbench.go | 7 +++++-- tools/gasbench/programs.go | 11 ++++++++-- 7 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 tools/gasbench/diff_test.go diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index 0d07c1cbef..dd796b72ba 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -49,10 +49,11 @@ families are special-cased in `BuildCaseWith`: - **Stack ops** (`DUP1`, `SWAP1`) aren't "n operands → 1 result" — each gets its own construction. - **SHL/SHR/SAR** need two *distinct* operands, not `n` copies of the same - value: a shift amount ≥256 takes go-ethereum's `value.Clear()` early-out - (the cheapest possible case), so reusing one seed for both operands would - measure that early-out instead of the real limb-shift path. `seedShift` - keeps the shift amount in-range and distinct from the value operand. + value: an out-of-range shift amount takes go-ethereum's `value.Clear()` + early-out (the cheapest possible case) — at shift ≥256 for SHL/SHR, shift + >256 for SAR — so reusing one seed for both operands would measure that + early-out instead of the real limb-shift path. `seedShift` keeps the shift + amount in-range and distinct from the value operand. `EXP` and anything with dynamic/memory/state gas is out of scope — see `OpSpec.DataDependent` and the design's Non-goals. @@ -63,7 +64,8 @@ families are special-cased in `BuildCaseWith`: handler and no Sei `GasMeter`. `gasUsed` is pure EVM gas (`cfg.GasLimit - leftOverGas`); there is deliberately no Cosmos gas meter in this path. Correlate against **EVM** gas, not the Cosmos gas meter — mixing -the two units was the CON-368 trap (`msg_server.go:152`). +the two units was the CON-368 trap (`x/evm/keeper/msg_server.go`'s +`serverRes.GasUsed` assignment, which is in EVM's gas unit, not Sei's). ## Program reuse across calls (journal growth) @@ -81,9 +83,10 @@ estimator reads. **Re-check this before reusing a `Program` across an order-of-magnitude more iterations, or for a `Case` that touches persistent state (SSTORE/LOG/CREATE).** -This periodic reallocation is also the one CoV-inflating noise source the -`getrusage`-based diagnostics ("Active-benchmarking diagnostics" below) -structurally cannot see (it's not a scheduler preemption). +This periodic reallocation is also the one noise source that inflates the +coefficient of variation (CoV, stddev/mean — see "Acceptance gate" below) +without the `getrusage`-based diagnostics ("Active-benchmarking diagnostics" +further below) being able to see it (it's not a scheduler preemption). ## Error contract @@ -110,10 +113,11 @@ I-cache/D-cache/branch predictor, catch a broken program early), then ## Acceptance gate: Significant, not CoV -`Diff.Significant` (`diff.go`) — `|DeltaNs| > SigmaK * Uncertainty`, where +`Diff.Significant` (`diff.go`) — `|DeltaNs| > SigmaK * Uncertainty` (and +`Uncertainty > 0`; a series with fewer than 2 samples has nothing to +estimate variance from and must never read as significant), where `Uncertainty` is the two series' median standard errors propagated in -quadrature — is the acceptance gate. A raw per-series coefficient of -variation (CoV, stddev/mean) of several percent +quadrature — is the acceptance gate. A raw per-series CoV of several percent under plain core-pinning (no kernel-level isolcpus/nohz_full/rcu_nocbs) is expected physics (the periodic scheduler tick, device IRQs, shared-cache contention that pinning alone can't remove), not a defect. No major diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index b583f32a5d..84e3f745ea 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -24,7 +24,6 @@ func BenchmarkOpcodes(b *testing.B) { var runs []Run for _, c := range cases { - c := c b.Run(c.OpcodeID, func(b *testing.B) { baseProg, err := NewProgram(c.Baseline) if err != nil { diff --git a/tools/gasbench/diff.go b/tools/gasbench/diff.go index c2ed71042f..c305115bfd 100644 --- a/tools/gasbench/diff.go +++ b/tools/gasbench/diff.go @@ -67,7 +67,11 @@ func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covCei } d.DeltaNs = ts.Median - bs.Median d.Uncertainty = math.Hypot(ts.SEMedian, bs.SEMedian) - d.Significant = math.Abs(d.DeltaNs) > sigmaK*d.Uncertainty + // Uncertainty == 0 (e.g. Iterations < 2, or an improbable zero-variance + // sample) means no variance was ever estimated, not that the delta is + // perfectly known -- Significant must never be true from an + // unestimated Uncertainty. + d.Significant = d.Uncertainty > 0 && math.Abs(d.DeltaNs) > sigmaK*d.Uncertainty d.HighVariance = bs.CoV > covCeiling || ts.CoV > covCeiling if target.GasUsed >= baseline.GasUsed { diff --git a/tools/gasbench/diff_test.go b/tools/gasbench/diff_test.go new file mode 100644 index 0000000000..650626a8a8 --- /dev/null +++ b/tools/gasbench/diff_test.go @@ -0,0 +1,40 @@ +package gasbench + +import ( + "testing" + "time" +) + +// TestSubtractNeverSignificantWithZeroUncertainty pins that a series with an +// unestimated variance (fewer than 2 samples, or an improbable zero-variance +// sample) never yields Significant=true, however large the delta -- a raw +// |DeltaNs| > 0 must not stand in for a real effect-size check against an +// uncertainty that was never estimated. +func TestSubtractNeverSignificantWithZeroUncertainty(t *testing.T) { + low1 := Series{Samples: []time.Duration{100 * time.Nanosecond}} + high1 := Series{Samples: []time.Duration{100000 * time.Nanosecond}} + low2 := Series{Samples: []time.Duration{100 * time.Nanosecond, 100 * time.Nanosecond}} + high2 := Series{Samples: []time.Duration{100000 * time.Nanosecond, 100000 * time.Nanosecond}} + + cases := []struct { + name string + baseline Series + target Series + }{ + {"single-sample baseline, zero-variance target", low1, high2}, + {"zero-variance baseline, single-sample target", low2, high1}, + {"both single-sample", low1, high1}, + {"zero-variance both", low2, high2}, + } + + for _, tc := range cases { + d := Subtract("ADD", tc.baseline, tc.target, 1, 3, 0.25) + if d.Uncertainty != 0 { + t.Fatalf("%s: Uncertainty = %v, want 0 for this fixture", tc.name, d.Uncertainty) + } + if d.Significant { + t.Errorf("%s: Significant = true with Uncertainty == 0 and DeltaNs = %v -- an unestimated uncertainty must never pass the gate", + tc.name, d.DeltaNs) + } + } +} diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index 601ba13c7f..32112a6ecb 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -91,6 +91,10 @@ func WriteCSV(w io.Writer, runs []Run) error { strconv.FormatFloat(r.ExecTimeNs, 'g', -1, 64), r.Status, strconv.Itoa(r.Iterations), + // CoV is advisory-only (README.md "Acceptance gate"), so this + // deliberately rounds to 6 sig figs for readability; NDJSON's + // json.Encoder gives CoV full precision instead -- a consumer + // comparing CoV across the two formats will see this difference. strconv.FormatFloat(r.CoV, 'g', 6, 64), strconv.FormatBool(r.Significant), strconv.FormatBool(r.HighVariance), diff --git a/tools/gasbench/gasbench.go b/tools/gasbench/gasbench.go index f84dc4854b..97643878ad 100644 --- a/tools/gasbench/gasbench.go +++ b/tools/gasbench/gasbench.go @@ -27,8 +27,11 @@ import ( // allocations across warmup+Iterations must fit in RAM. type RunOnce func() (gasUsed uint64, err error) -// sink defeats dead-code elimination of the gas result. A single measurement -// goroutine (GOMAXPROCS=1, locked thread) means no synchronization is needed. +// sink defeats dead-code elimination of the gas result. Safe without +// synchronization only because BenchmarkOpcodes's subtests run sequentially +// (no b.Parallel) -- GOMAXPROCS=1 and thread-locking do not by themselves +// make the ^= below race-free, and do not protect this var if a future +// change parallelizes the subtests. var sink uint64 // Config controls one measurement series. diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index e7a2871966..18c73f6476 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -1,6 +1,8 @@ package gasbench import ( + "fmt" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/params" @@ -14,6 +16,7 @@ const DefaultReps = 1000 // Class groups opcodes by arithmetic character. type Class string +// Opcode classes. const ( ClassArithmetic Class = "arithmetic" ClassBitwise Class = "bitwise" @@ -133,8 +136,7 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { case vm.SHL, vm.SHR, vm.SAR: // DUP2 (not DUP1 x2, unlike the default branch below): these ops // need two DISTINCT operands, not n copies of one value -- see - // README.md "Differential construction" (the value.Clear() early-out - // a same-value construction would accidentally measure). + // README.md "Differential construction". base.Push(seed).Push(seedShift) tgt.Push(seed).Push(seedShift) for i := 0; i < reps; i++ { @@ -146,6 +148,11 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { default: // General n-operand case: see README.md "Differential construction" // for the (n-1)*GasQuickStep derivation this formula implements. + // Arity 0 must be special-cased above (like DUP1/SWAP1): the + // (n-1)*GasQuickStep term below assumes n >= 1. + if s.Arity < 1 { + panic(fmt.Sprintf("gasbench: %s has Arity %d < 1 and is not special-cased in BuildCaseWith", s.Name, s.Arity)) + } base.Push(seed).Push(seed) tgt.Push(seed).Push(seed) for i := 0; i < reps; i++ { From a5d0b80bec061db60e02017b7c6f88a607c603ef Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 17:32:46 -0700 Subject: [PATCH 08/19] test(tools/gasbench): close xreview R4 advisory items All four R4 reviewers ratified; these are their advisory-tier items: - diff_test.go: t.Errorf+continue instead of t.Fatalf inside the table loop, so a broken fixture doesn't mask whether sibling cases also regressed (idiomatic-reviewer, per the keep-going table convention emit_test.go established). - programs_test.go: new recover-based test pinning BuildCaseWith's Arity<1 panic guard, previously the only invariant in the package with no unit test (it is dead code for the in-tree Specs, so only a test can exercise it). - README.md: the SHL/SHR/SAR early-out is value.Clear() -- or SetAllOne() for SAR on a negative operand (solidity-developer, verified against the fork's opSAR). Local branch only, not pushed. --- tools/gasbench/README.md | 7 ++++--- tools/gasbench/diff_test.go | 3 ++- tools/gasbench/programs_test.go | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 tools/gasbench/programs_test.go diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index dd796b72ba..4a553bf03f 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -49,9 +49,10 @@ families are special-cased in `BuildCaseWith`: - **Stack ops** (`DUP1`, `SWAP1`) aren't "n operands → 1 result" — each gets its own construction. - **SHL/SHR/SAR** need two *distinct* operands, not `n` copies of the same - value: an out-of-range shift amount takes go-ethereum's `value.Clear()` - early-out (the cheapest possible case) — at shift ≥256 for SHL/SHR, shift - >256 for SAR — so reusing one seed for both operands would measure that + value: an out-of-range shift amount takes go-ethereum's constant-time + early-out (the cheapest possible case; `value.Clear()`, or `SetAllOne()` + for SAR on a negative operand) — at shift ≥256 for SHL/SHR, shift >256 + for SAR — so reusing one seed for both operands would measure that early-out instead of the real limb-shift path. `seedShift` keeps the shift amount in-range and distinct from the value operand. diff --git a/tools/gasbench/diff_test.go b/tools/gasbench/diff_test.go index 650626a8a8..28d7b26716 100644 --- a/tools/gasbench/diff_test.go +++ b/tools/gasbench/diff_test.go @@ -30,7 +30,8 @@ func TestSubtractNeverSignificantWithZeroUncertainty(t *testing.T) { for _, tc := range cases { d := Subtract("ADD", tc.baseline, tc.target, 1, 3, 0.25) if d.Uncertainty != 0 { - t.Fatalf("%s: Uncertainty = %v, want 0 for this fixture", tc.name, d.Uncertainty) + t.Errorf("%s: Uncertainty = %v, want 0 for this fixture", tc.name, d.Uncertainty) + continue } if d.Significant { t.Errorf("%s: Significant = true with Uncertainty == 0 and DeltaNs = %v -- an unestimated uncertainty must never pass the gate", diff --git a/tools/gasbench/programs_test.go b/tools/gasbench/programs_test.go new file mode 100644 index 0000000000..ef7cfcf20c --- /dev/null +++ b/tools/gasbench/programs_test.go @@ -0,0 +1,20 @@ +package gasbench + +import ( + "testing" + + "github.com/ethereum/go-ethereum/core/vm" +) + +// TestBuildCaseWithPanicsOnUnhandledArityZero pins the default branch's +// Arity guard: an Arity-0 spec that is not special-cased (unlike DUP1/SWAP1) +// must panic rather than silently feed n=0 into the (n-1)*GasQuickStep +// delta formula. +func TestBuildCaseWithPanicsOnUnhandledArityZero(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("BuildCaseWith accepted an Arity-0 spec in the default branch; want panic") + } + }() + BuildCaseWith(OpSpec{Name: "PC", Op: vm.PC, Class: ClassStack, Arity: 0, ConstGas: vm.GasQuickStep}, 10, seed256) +} From 5d007fb44572729fdd61442d2cec8ca8928c367c Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 19:18:15 -0700 Subject: [PATCH 09/19] docs(tools/gasbench): add operator Quickstart runbook to README A handoff-ready walkthrough at the top of README.md: smoke-run command, what a trustworthy host looks like, how to interpret the output (per-op time, per-op gas, and ns-per-gas as the hypothesis metric -- constant across opcodes if pricing tracked time, spread = the mispricing signal), and when to trust a row (significant / high_variance / nivcsw / cross-count agreement). AGENTS.md points agents at it; README points operators at AGENTS.md for iteration rules. Reviewed by systems-engineer (every claim vs the code) and prose-steward (standalone dual-audience legibility); their findings folded in: - The confidence-check instruction originally pointed at the CSV, but each -count rerun rewrites the output files, so the CSV holds only the final count's rows (os.Create truncation per BenchmarkOpcodes invocation -- verified). The runbook now pipes run.sh through tee and directs the cross-count agreement check at the captured benchmark stdout (benchstat-consumable), which is where the per-count history actually lives. - emit.go's StatusInsignificant doc claimed a quieter host doesn't help; it does (lower Stddev -> lower SEMedian). Reconciled. - "cancels host-speed effects" softened to "cancels the common clock-speed factor" -- microarchitectural ratios still differ across hosts, which is why repricing-grade numbers come from the dedicated host. - nivcsw trust rule now defines both sides of the discriminator (nonzero -> host preemption; near-zero + high CoV -> the harness's own non-scheduler tail). - Tied status=insignificant and significant=false as one condition; step-lead grammar made uniform; gas_used positivity noted for the ns-per-gas formula. Local branch only, not pushed. --- tools/gasbench/AGENTS.md | 7 ++-- tools/gasbench/README.md | 75 ++++++++++++++++++++++++++++++++++++++++ tools/gasbench/emit.go | 2 +- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/tools/gasbench/AGENTS.md b/tools/gasbench/AGENTS.md index 98c140566f..68f517635f 100644 --- a/tools/gasbench/AGENTS.md +++ b/tools/gasbench/AGENTS.md @@ -38,8 +38,9 @@ file is the short orientation. tools/gasbench/run.sh ``` -Env vars, output schema, and the `-count` semantics caveat: see README.md -"Running it" / "Output schema". +End-to-end walkthrough incl. how to interpret the output: README.md +"Quickstart". Env vars, output schema, and the `-count` semantics caveat: +README.md "Running it" / "Output schema". ## Files @@ -54,5 +55,5 @@ Env vars, output schema, and the `-count` semantics caveat: see README.md | `emit_test.go` | pins `Run.Status` as a pure function of `Diff.Significant` | | `bench_test.go` | `BenchmarkOpcodes`: wires the above into `go test -bench` | | `run.sh` | pinned-core runner + operator checklist for turbo/governor/isolation | -| `README.md` | full rationale: construction, acceptance gate, diagnostics | +| `README.md` | operator quickstart + full rationale: construction, acceptance gate, diagnostics | | `AGENTS.md` | this file | diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index 4a553bf03f..e404133235 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -9,6 +9,81 @@ the `bdchatham-designs` repo. Noise-floor calibration behind the acceptance gate below: `designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md` in the same repo. +## Quickstart + +To iterate on this harness with an agent, point it at this directory's +`AGENTS.md` first — it carries the rules a change here must not break. + +**1. Smoke run (any machine, indicative numbers only):** + +```bash +cd sei-chain +GASBENCH_OUT_CSV=gasbench.csv GASBENCH_OUT_NDJSON=gasbench.ndjson \ + tools/gasbench/run.sh | tee gasbench.log +``` + +This benchmarks every scalar opcode in `Specs` and writes one row per opcode +to the output files. Relative output paths land in `tools/gasbench/` (the +test binary's working directory), not where you invoked `run.sh` — pass +absolute paths to put them elsewhere. Keep the `tee`: the CSV/NDJSON hold +only the *final* count's rows (each of `-count`'s reruns rewrites the +files), so the per-count history needed for step 4's agreement check exists +only in the benchmark stdout you just captured. Several minutes at the +defaults; set `GASBENCH_ITERS=2000 GASBENCH_COUNT=1` for a fast first look. +On a laptop or shared box, `run.sh` will warn that results are indicative: +expect cheap opcodes (`ADD`'s per-op cost is ~1ns, a whole-program delta +close to the noise) to come back `status=insignificant` there, and don't +read anything into them. + +**2. For real numbers, use a quiet, dedicated Linux host:** bare metal (no +co-tenants), fixed or pinned CPU frequency, one core reserved for the run +(`GASBENCH_CORE`). `run.sh`'s header comment is the operator checklist. A +fixed-frequency ARM server instance (e.g. a Graviton `.metal`) needs the +least setup because there is no turbo/governor to disable. Kernel-level +isolation (`isolcpus` etc.) is deliberately NOT required — see "Acceptance +gate" below for why plain pinning is enough. + +**3. Read the output:** each row is one opcode's *differential* cost — the +target-minus-baseline delta, so setup/loop overhead has already cancelled +(see "Differential construction"): + +- **per-op time** = `exec_time_ns / reps`; **per-op gas** = `gas_used / reps`. +- **ns-per-gas** = `exec_time_ns / gas_used` (`gas_used` is positive for + every in-scope opcode) — the number the hypothesis test is about. If gas + pricing tracked execution time, ns-per-gas would be roughly constant + across opcodes; the spread across rows IS the mispricing signal. Comparing + rows within one run cancels the common clock-speed factor, so the spread + is meaningful even though the absolute nanoseconds are host-specific — + but microarchitectural ratios (adder vs shifter vs multiplier cost) still + differ between hosts, which is why repricing-grade numbers come from the + step-2 host, not a laptop. + +**4. Only trust a row that earns it:** + +- `significant=false` (the row's `status` reads `insignificant` — same + condition, derived field): the delta didn't clear its own measurement + uncertainty — treat the time as indistinguishable from zero at this + precision. Raise `GASBENCH_ITERS`, or move to the quiet host; don't + average or rank insignificant rows. +- `high_variance=true`: advisory — the run saw unusual dispersion (noisy + neighbor, throttling). It does not invalidate a significant result, and + `nivcsw` tells you where to look: nonzero means the scheduler preempted + the process mid-window (blame the host); near-zero alongside the high + CoV points at a non-scheduler tail inside the harness itself (see + "Active-benchmarking diagnostics"). +- Rerun-to-rerun agreement is the real confidence check: `run.sh` defaults + to `-count=10`, and the per-count `per-op-ns`/`per-op-gas` metric lines + live in the captured stdout (`gasbench.log` from step 1 — `benchstat` + consumes that format directly), not in the CSV. A per-op delta that is + stable across counts (and across invocations) is the result. See + "Running it" for what `-count` does and doesn't give you. + +**Scope:** scalar constant-gas opcodes only (arithmetic/bitwise/comparison/ +stack). Parametric opcodes (`KECCAK256`/`EXP`/copies), the state-touching +matrix (`SLOAD`/`SSTORE`/`CALL`), and real-block replay are deferred — see +"Scope" at the bottom for the un-defer triggers. To add an opcode within +scope, follow `AGENTS.md` "New opcode specs". + ## The load-bearing invariant: never attach a tracer to a timed call Attaching a `tracing.Hooks` tracer to sum per-opcode gas sets `debug=true` in diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index 32112a6ecb..d29a50799b 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -29,7 +29,7 @@ type Run struct { // Significant-driven, not CoV-driven. const ( StatusOK = "ok" // Significant is true -- the delta is trustworthy regardless of routine CoV - StatusInsignificant = "insignificant" // Significant is false -- the delta does not clear its own measurement uncertainty; needs more Iterations or a coarser SigmaK, not a quieter host + StatusInsignificant = "insignificant" // Significant is false -- the delta does not clear its own measurement uncertainty; more Iterations or a quieter host shrink that uncertainty // StatusError is reserved for a per-case measurement failure. Not // currently produced: bench_test.go fails the whole benchmark loudly // (b.Fatalf) on an invalid program rather than degrading to an error From 9360ca66e9250e2b2208ac5e14fbcc74530e4bd5 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 14 Jul 2026 19:42:31 -0700 Subject: [PATCH 10/19] docs(tools/gasbench): correct -count file semantics in runbook, validated E2E The full E2E run on the dedicated c6g.metal host falsified the runbook's claim that each -count rerun rewrites the output files. Go applies -count to the leaf subtests inside a single BenchmarkOpcodes invocation, so runs accumulates across counts and the file is written once holding one row per opcode per count (verified on the host CSV: 220 rows for 22 opcodes at -count=10, and reproduced locally with -count=3). The cross-count agreement check therefore reads straight from the CSV; the teed stdout is the benchstat-consumable form of the same data, now optional. --- tools/gasbench/README.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index e404133235..b27e235192 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -22,13 +22,15 @@ GASBENCH_OUT_CSV=gasbench.csv GASBENCH_OUT_NDJSON=gasbench.ndjson \ tools/gasbench/run.sh | tee gasbench.log ``` -This benchmarks every scalar opcode in `Specs` and writes one row per opcode -to the output files. Relative output paths land in `tools/gasbench/` (the -test binary's working directory), not where you invoked `run.sh` — pass -absolute paths to put them elsewhere. Keep the `tee`: the CSV/NDJSON hold -only the *final* count's rows (each of `-count`'s reruns rewrites the -files), so the per-count history needed for step 4's agreement check exists -only in the benchmark stdout you just captured. Several minutes at the +This benchmarks every scalar opcode in `Specs` and writes one row per +opcode *per count* to the output files (`-count=10` by default, so 10 rows +per opcode — `-count` multiplies the subtests inside one benchmark +invocation, and the file is written once at the end with all of them). +Relative output paths land in `tools/gasbench/` (the test binary's working +directory), not where you invoked `run.sh` — pass absolute paths to put +them elsewhere. The `tee` is optional: the CSV already carries the +cross-count history step 4 needs; the captured stdout is the +`benchstat`-consumable form of the same data. Roughly ten minutes at the defaults; set `GASBENCH_ITERS=2000 GASBENCH_COUNT=1` for a fast first look. On a laptop or shared box, `run.sh` will warn that results are indicative: expect cheap opcodes (`ADD`'s per-op cost is ~1ns, a whole-program delta @@ -72,10 +74,11 @@ target-minus-baseline delta, so setup/loop overhead has already cancelled CoV points at a non-scheduler tail inside the harness itself (see "Active-benchmarking diagnostics"). - Rerun-to-rerun agreement is the real confidence check: `run.sh` defaults - to `-count=10`, and the per-count `per-op-ns`/`per-op-gas` metric lines - live in the captured stdout (`gasbench.log` from step 1 — `benchstat` - consumes that format directly), not in the CSV. A per-op delta that is - stable across counts (and across invocations) is the result. See + to `-count=10`, so the CSV has 10 rows per opcode — group by `input_id` + and look at the spread of `exec_time_ns` across counts (the same data is + in `gasbench.log` in `benchstat`-consumable form). A per-op delta that is + stable across counts (and across invocations) is the result; on a + dedicated pinned host, expect low-single-digit-percent spread. See "Running it" for what `-count` does and doesn't give you. **Scope:** scalar constant-gas opcodes only (arithmetic/bitwise/comparison/ @@ -244,10 +247,11 @@ go test ./tools/gasbench/ -run '^$' -bench '^BenchmarkOpcodes$' \ ``` `-benchtime=1x` matters: `BenchmarkOpcodes`'s inner timing loop is -`Measure`'s, not `b.N`'s, so `b.N` must stay at 1. `-count=K` reruns the -benchmark K times **within the same OS process**, not as K independent -process forks — it gives `benchstat`-style cross-run variance, not -process-level independence. True process-level independence would need K +`Measure`'s, not `b.N`'s, so `b.N` must stay at 1. `-count=K` reruns each +opcode subtest K times **within the same OS process** (inside a single +`BenchmarkOpcodes` invocation, so the output files accumulate all K rows +per opcode), not as K independent process forks — it gives +`benchstat`-style cross-run variance, not process-level independence. True process-level independence would need K separate `go test` invocations. See the research doc for why this matters less than it sounds: JMH's fork/warmup/measurement-iteration model is the gold standard, but this MVP's within-process reruns already surfaced From fb4cb35b8374f65f66a7af3008cbbce057a69be1 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 08:24:57 -0700 Subject: [PATCH 11/19] fix(tools/gasbench): distinct operands per case; emit nominal const_gas Two correctness fixes to the differential construction, validated E2E on a dedicated c6g.metal (220/220 significant, all gas self-checks green): - Feed each case distinct ascending operands (2^192-1, 2^224-1, 2^256-1) lifted via DUP, instead of n copies of one seed via DUP1. Equal operands let holiman/uint256 short-circuit DIV(x,x)->1 and MOD(x,x)->0 before udivrem, so those rows timed a compare instead of a 256-bit division (~10x understated, inverting their place in the spread). The gas algebra is unchanged (every DUP is GasFastestStep); the per-count gas self-check still passes on all 22 opcodes. - Emit the nominal jump-table gas as const_gas alongside the differential gas_used. ns-per-gas divides by const_gas (what the chain charges); the differential denominator is deflated by an arity-correlated factor and would report a spread even for perfectly-priced opcodes. Corrected headline on the pinned host: ns-per-nominal-gas spans 1.05 (DUP1) to 27.2 (MULMOD), ~26x, with DIV/MOD/MULMOD correctly at the expensive end. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/README.md | 73 +++++++++++++++++++---------- tools/gasbench/emit.go | 7 ++- tools/gasbench/programs.go | 82 ++++++++++++++++++++++++--------- tools/gasbench/programs_test.go | 2 +- 4 files changed, 116 insertions(+), 48 deletions(-) diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index b27e235192..d1c2c15281 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -49,16 +49,27 @@ gate" below for why plain pinning is enough. target-minus-baseline delta, so setup/loop overhead has already cancelled (see "Differential construction"): -- **per-op time** = `exec_time_ns / reps`; **per-op gas** = `gas_used / reps`. -- **ns-per-gas** = `exec_time_ns / gas_used` (`gas_used` is positive for - every in-scope opcode) — the number the hypothesis test is about. If gas - pricing tracked execution time, ns-per-gas would be roughly constant - across opcodes; the spread across rows IS the mispricing signal. Comparing - rows within one run cancels the common clock-speed factor, so the spread - is meaningful even though the absolute nanoseconds are host-specific — - but microarchitectural ratios (adder vs shifter vs multiplier cost) still - differ between hosts, which is why repricing-grade numbers come from the - step-2 host, not a laptop. +- **per-op time** = `exec_time_ns / reps`. +- **two gas columns, and they are not the same number.** `const_gas` is the + nominal gas the chain actually charges for the opcode (the jump-table + constant: ADD=3, MULMOD=8). `gas_used` is the *differential* whole-program + delta the harness measured — net of the filler the baseline leaves behind — + so it is smaller (ADD→1, MULMOD→4) and arity-dependent. `gas_used` exists to + drive the construction self-check; **`const_gas` is the denominator for any + repricing statement.** +- **ns-per-gas** = `exec_time_ns / reps / const_gas` — the number the + hypothesis test is about: time per gas-the-chain-charges. If pricing tracked + execution time it would be roughly constant across opcodes; the spread IS the + mispricing signal. Do NOT divide by `gas_used` — that differential denominator + is deflated by an arity-correlated factor (up to 3x) and would report a spread + even for a perfectly-priced instruction set. Quote the spread only across + `significant=true` rows: the cheap ops (DUP1, ADD…) are dominated by the + op-minus-filler marginal and routinely come back `insignificant` on a normal + host — do not anchor the low end of the spread on one. Comparing rows within + one run cancels the common clock-speed factor, so the spread is meaningful + even though the absolute nanoseconds are host-specific — but microarchitectural + ratios (adder vs shifter vs multiplier cost) still differ between hosts, which + is why repricing-grade numbers come from the step-2 host, not a laptop. **4. Only trust a row that earns it:** @@ -113,26 +124,39 @@ programs share — loop overhead, setup, dispatch cost — cancels when the opcode. The repeating unit is balanced so gas is net-zero and the stack never grows: -for an opcode consuming `n` operands and producing 1 result, +for an opcode consuming `n` operands and producing 1 result, `n` *distinct* +operands sit at the base of the stack and each unit lifts fresh copies of all +`n` with `DUP` (`DUP` copies the deepest of the `n`; repeated `n` times +it re-lifts the whole tuple in order): ``` -target = DUP1 x n , OP , POP // dup n copies, run op, drop result -baseline = DUP1 x n , POP x n // dup n copies, drop them +setup = PUSH v0 .. PUSH v_{n-1} // n distinct operands, once +target = DUP x n , OP , POP // lift the n-tuple, run op, drop result +baseline = DUP x n , POP x n // lift the n-tuple, drop them ``` -The `n` DUP1s cancel, one POP cancels the target's trailing POP, so the -per-unit gas delta is `ConstGas(op) - (n-1)*GasQuickStep`. Two opcode -families are special-cased in `BuildCaseWith`: +The `n` DUPs cancel, one POP cancels the target's trailing POP, so the +per-unit gas delta is `ConstGas(op) - (n-1)*GasQuickStep` (every `DUP` is +`GasFastestStep`, so the fill choice doesn't change the algebra). + +**The operands must be distinct, and this is load-bearing, not cosmetic.** +`holiman/uint256` short-circuits degenerate operands before the real kernel: +`DIV(x,x)→1` and `MOD(x,x)→0` return without a division, so feeding `n` copies +of one value (an earlier version's `DUP1 x n`) would time an `Eq` compare, not +a 256-bit divide — understating DIV/MOD by ~10x and inverting their place in +the spread. `seedOperands` is ascending so the arity-2 dividend (top) stays +above the divisor and DIV/MOD reach `udivrem`, and the smallest value lands at +the modulus slot so ADDMOD/MULMOD operands aren't pre-reduced. Two opcode +families are still special-cased in `BuildCaseWith`: - **Stack ops** (`DUP1`, `SWAP1`) aren't "n operands → 1 result" — each gets its own construction. -- **SHL/SHR/SAR** need two *distinct* operands, not `n` copies of the same - value: an out-of-range shift amount takes go-ethereum's constant-time - early-out (the cheapest possible case; `value.Clear()`, or `SetAllOne()` - for SAR on a negative operand) — at shift ≥256 for SHL/SHR, shift >256 - for SAR — so reusing one seed for both operands would measure that - early-out instead of the real limb-shift path. `seedShift` keeps the shift - amount in-range and distinct from the value operand. +- **SHL/SHR/SAR** need a small *in-range* shift amount on top (not a full-width + value): an out-of-range shift takes go-ethereum's constant-time early-out + (the cheapest case; `value.Clear()`, or `SetAllOne()` for SAR on a negative + operand) — at shift ≥256 for SHL/SHR, shift >256 for SAR — so a full-width + top operand would measure that early-out instead of the real limb-shift path. + `seedShift` keeps the shift amount in-range and distinct from the value. `EXP` and anything with dynamic/memory/state gas is out of scope — see `OpSpec.DataDependent` and the design's Non-goals. @@ -270,7 +294,8 @@ One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON: | `input_id` | opcode id, e.g. `ADD` | | `class` | opcode family, e.g. `arithmetic` | | `reps` | opcode executions the delta represents | -| `gas_used` | whole-program gas delta (target - baseline); per-op = `gas_used/reps` | +| `gas_used` | *differential* whole-program gas delta (target - baseline), net of filler; per-op = `gas_used/reps`. Drives the construction self-check — NOT the repricing denominator | +| `const_gas` | nominal per-op gas the chain charges (jump-table `ConstGas`); the denominator for ns-per-gas — see "Read the output" | | `exec_time_ns` | whole-program time delta, ns; per-op = `exec_time_ns/reps` | | `status` | `ok` if `significant`, else `insignificant` — never gated on CoV | | `iterations` | timed iterations behind each series | diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index d29a50799b..e50c74d326 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -14,7 +14,8 @@ type Run struct { InputID string `json:"input_id"` // opcode id, e.g. "ADD" Class string `json:"class"` // opcode family, e.g. "arithmetic" Reps int `json:"reps"` // opcode executions the delta represents - GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps + GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps -- this is the DIFFERENTIAL (net of filler), not the gas the chain charges + ConstGas uint64 `json:"const_gas"` // nominal per-op gas the chain charges (jump-table ConstGas); the repricing-relevant denominator for ns-per-gas. See README.md "Read the output" ExecTimeNs float64 `json:"exec_time_ns"` // whole-program time delta, ns (target median - baseline median); per-op = ExecTimeNs/Reps Status string `json:"status"` // ok if Significant, else insignificant -- never gated on CoV Iterations int `json:"iterations"` // timed iterations behind each series @@ -63,6 +64,7 @@ func NewRun(c Case, d Diff, iterations int) Run { Class: string(c.Class), Reps: d.Reps, GasUsed: d.GasDelta, + ConstGas: c.ConstGas, ExecTimeNs: d.DeltaNs, Status: status, Iterations: iterations, @@ -74,7 +76,7 @@ func NewRun(c Case, d Diff, iterations int) Run { } } -var csvHeader = []string{"input_id", "class", "reps", "gas_used", "exec_time_ns", "status", "iterations", "cov", "significant", "high_variance", "nvcsw", "nivcsw"} +var csvHeader = []string{"input_id", "class", "reps", "gas_used", "const_gas", "exec_time_ns", "status", "iterations", "cov", "significant", "high_variance", "nvcsw", "nivcsw"} // WriteCSV writes a header plus one row per run. func WriteCSV(w io.Writer, runs []Run) error { @@ -88,6 +90,7 @@ func WriteCSV(w io.Writer, runs []Run) error { r.Class, strconv.Itoa(r.Reps), strconv.FormatUint(r.GasUsed, 10), + strconv.FormatUint(r.ConstGas, 10), strconv.FormatFloat(r.ExecTimeNs, 'g', -1, 64), r.Status, strconv.Itoa(r.Iterations), diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index 18c73f6476..1563d20064 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -82,25 +82,43 @@ type Case struct { Class Class DataDependent bool Reps int + ConstGas uint64 // nominal per-op gas the chain charges (OpSpec.ConstGas); the repricing-relevant denominator, distinct from the differential GasDelta Baseline []byte Target []byte ExpectedGasDelta uint64 } -// seed256 is the fixed working-value operand: full-width (2^256-1) so -// magnitude-dependent ops exercise their full uint256 limb path. -var seed256 = new(uint256.Int).Not(uint256.NewInt(0)) +// seedOperands are the distinct, non-zero, ascending working-value operands +// fed to the general (default) branch -- one per stack slot the opcode +// consumes, index 0 landing at the modulus/divisor position and the last at +// the top of the stack. +// +// Distinctness and order are load-bearing, not cosmetic. Equal operands make +// holiman/uint256 short-circuit the arithmetic before the real kernel: +// DIV(x,x) returns 1 and MOD(x,x) returns 0 without ever calling udivrem +// (uint256.go Div/Mod), so an all-equal input would time a compare, not a +// 256-bit division, and understate DIV/MOD by ~10x. Ascending order keeps the +// arity-2 dividend (top) above the divisor so DIV/MOD reach udivrem, and puts +// the smallest value at the modulus slot for ADDMOD/MULMOD so their operands +// are not already reduced. All three are multi-limb; the largest is full-width. +// See README.md "Differential construction". +var seedOperands = []*uint256.Int{ + uint256.MustFromHex("0xffffffffffffffffffffffffffffffffffffffffffffffff"), // 2^192-1 (divisor/modulus slot) + uint256.MustFromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // 2^224-1 + uint256.MustFromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // 2^256-1 (full width) +} -// seedShift is a fixed, in-range (<256) shift amount for SHL/SHR/SAR -- -// see README.md "Differential construction" for why this must differ from -// seed256. +// seedShift is a fixed, in-range (<256) shift amount for SHL/SHR/SAR -- it must +// be a small distinct operand, not a full-width value, or the shift would hit +// go-ethereum's "shift >= 256 -> 0" early-out. See README.md "Differential +// construction". var seedShift = uint256.NewInt(4) // BuildCases builds every spec's case at DefaultReps. func BuildCases() []Case { out := make([]Case, 0, len(Specs)) for _, s := range Specs { - out = append(out, BuildCaseWith(s, DefaultReps, seed256)) + out = append(out, BuildCaseWith(s, DefaultReps, seedOperands)) } return out } @@ -109,15 +127,23 @@ func BuildCases() []Case { // README.md "Differential construction" for the balanced-DUP/POP algebra; // per-branch delta formulas are inlined below next to the code they justify. // Every unit is net-0 so stack depth never grows or underflows. -func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { +func BuildCaseWith(s OpSpec, reps int, operands []*uint256.Int) Case { base := program.New() tgt := program.New() + // DUP1/SWAP1 need two values on the stack; the general branch needs one + // distinct operand per slot the opcode consumes. Guard here so a caller + // that under-supplies operands fails at construction, not with a silent + // stack underflow at measurement. + if len(operands) < 2 || len(operands) < s.Arity { + panic(fmt.Sprintf("gasbench: %s needs >= max(2, arity=%d) operands, got %d", s.Name, s.Arity, len(operands))) + } + var perUnitDelta uint64 switch s.Op { case vm.DUP1: - base.Push(seed).Push(seed) - tgt.Push(seed).Push(seed) + base.Push(operands[0]).Push(operands[1]) + tgt.Push(operands[0]).Push(operands[1]) for i := 0; i < reps; i++ { tgt.Op(vm.DUP1, vm.POP) base.Op(vm.PUSH0, vm.POP) @@ -125,8 +151,8 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { perUnitDelta = s.ConstGas - vm.GasQuickStep // 3 - 2 case vm.SWAP1: - base.Push(seed).Push(seed) - tgt.Push(seed).Push(seed) + base.Push(operands[0]).Push(operands[1]) + tgt.Push(operands[0]).Push(operands[1]) for i := 0; i < reps; i++ { tgt.Op(vm.SWAP1) base.Op(vm.JUMPDEST) @@ -134,11 +160,14 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { perUnitDelta = s.ConstGas - params.JumpdestGas // 3 - 1 case vm.SHL, vm.SHR, vm.SAR: - // DUP2 (not DUP1 x2, unlike the default branch below): these ops - // need two DISTINCT operands, not n copies of one value -- see - // README.md "Differential construction". - base.Push(seed).Push(seedShift) - tgt.Push(seed).Push(seedShift) + // Shifts need a small in-range shift amount on top and a full-width + // value beneath -- distinct operands, or the shift hits go-ethereum's + // ">= 256 -> 0" early-out. This is the arity-2 shape of the default + // branch with a bespoke top operand; see README.md "Differential + // construction". + value := operands[len(operands)-1] // full-width + base.Push(value).Push(seedShift) + tgt.Push(value).Push(seedShift) for i := 0; i < reps; i++ { tgt.Op(vm.DUP2, vm.DUP2, s.Op, vm.POP) base.Op(vm.DUP2, vm.DUP2, vm.POP, vm.POP) @@ -153,12 +182,22 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { if s.Arity < 1 { panic(fmt.Sprintf("gasbench: %s has Arity %d < 1 and is not special-cased in BuildCaseWith", s.Name, s.Arity)) } - base.Push(seed).Push(seed) - tgt.Push(seed).Push(seed) + // Push one DISTINCT operand per slot, then DUP reps times. + // DUP copies the deepest of the arity operands; repeating it + // arity times lifts fresh copies of all arity operands back to the top + // in order, so every unit re-runs the op on the SAME distinct tuple + // (equal operands would let uint256 short-circuit DIV/MOD -- see + // seedOperands). GasFastestStep is identical for every DUP, so this + // leaves the (n-1)*GasQuickStep gas algebra unchanged from a DUP1 fill. + dupN := vm.DUP1 + vm.OpCode(s.Arity-1) + for d := 0; d < s.Arity; d++ { + base.Push(operands[d]) + tgt.Push(operands[d]) + } for i := 0; i < reps; i++ { for d := 0; d < s.Arity; d++ { - tgt.Op(vm.DUP1) - base.Op(vm.DUP1) + tgt.Op(dupN) + base.Op(dupN) } tgt.Op(s.Op, vm.POP) for d := 0; d < s.Arity; d++ { @@ -176,6 +215,7 @@ func BuildCaseWith(s OpSpec, reps int, seed *uint256.Int) Case { Class: s.Class, DataDependent: s.DataDependent, Reps: reps, + ConstGas: s.ConstGas, Baseline: base.Bytes(), Target: tgt.Bytes(), ExpectedGasDelta: perUnitDelta * uint64(reps), diff --git a/tools/gasbench/programs_test.go b/tools/gasbench/programs_test.go index ef7cfcf20c..e47da5c003 100644 --- a/tools/gasbench/programs_test.go +++ b/tools/gasbench/programs_test.go @@ -16,5 +16,5 @@ func TestBuildCaseWithPanicsOnUnhandledArityZero(t *testing.T) { t.Error("BuildCaseWith accepted an Arity-0 spec in the default branch; want panic") } }() - BuildCaseWith(OpSpec{Name: "PC", Op: vm.PC, Class: ClassStack, Arity: 0, ConstGas: vm.GasQuickStep}, 10, seed256) + BuildCaseWith(OpSpec{Name: "PC", Op: vm.PC, Class: ClassStack, Arity: 0, ConstGas: vm.GasQuickStep}, 10, seedOperands) } From 5eb438f47a28eb56365b31f5c8f7f7123af7ddb6 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 08:25:48 -0700 Subject: [PATCH 12/19] feat(tools/gasbench): cross-run statistics on x/perf/benchmath; effect-size floor Replace the hand-rolled significance machinery with golang.org/x/perf/benchmath (now a direct dep) and gate acceptance on statistics applied at the layer they are designed for: - The acceptance gate is the PAIRED per-count delta CI: distinguishable = the order-statistic CI on the -count delta series excludes zero (benchmath.AssumeNothing.Summary). Baseline/target are measured back-to-back per count, so the paired delta cancels common-run drift the prior 3-sigma gate (and any unpaired test) could not. The unpaired Mann-Whitney p (AssumeNothing.Compare) is emitted as p_value, advisory only. Deletes SEMedian/SEMean (a normal-theory estimator on non-normal latency data), the Hypot quadrature, SigmaK, Significant, and GASBENCH_SIGMA_K. - One aggregate row per opcode: -count passes accumulate in-process (opcodeAccum) and benchmath folds them into the row's CI; the per-count rows and the benchstat-over-stdout step are retired. New columns: count, ci_lo/ci_hi (null when underpowered; +/-Inf never reaches encoding/json), confidence, distinguishable, effect_size_pass, p_value, alpha. - Effect-size floor (GASBENCH_MIN_PEROP_NS, default 1.0ns, <=0 disables): status=ok requires distinguishable AND the per-op median delta clearing the floor, so "significant" means "meaningfully different," not merely distinguishable at n=20000. Distinguishable-but-below-floor rows read sub_threshold (NOT "correctly priced"); only ok is correlation-eligible. New statuses: sub_threshold, underpowered (count<2 or non-finite CI). - The per-count gas self-check is kept and strengthened with a gas-delta-identical-across-counts guard. The differential construction (programs.go) and the Measure loop (gasbench.go) are untouched. - Harness hygiene: run.sh warns when it cannot verify turbo/governor (ARM/non-Linux) instead of staying silent; output-file Close errors and unparseable GASBENCH_* env values fail loud. - README rewritten as the schema of record: paired-CI + effect-floor acceptance gate, full column table, GASBENCH_ALPHA/CI_CONFIDENCE/ MIN_PEROP_NS; cites EIP-7904 (anchor-rate framing) and Broken Metre (NDSS 2020) for the method and hypothesis lineage. Design and review ledger: bdchatham-designs designs/gas-repricing-telemetry/ gasbench-benchmath-integration.md (2 design rounds + 4 gated implementation increments, unanimous slate at each gate). Co-Authored-By: Claude Opus 4.8 (1M context) --- go.mod | 2 + go.sum | 2 + tools/gasbench/AGENTS.md | 16 +- tools/gasbench/README.md | 253 +++++++++++++++++++++++--------- tools/gasbench/bench_test.go | 170 +++++++++++++++------ tools/gasbench/crossrun.go | 59 ++++++++ tools/gasbench/crossrun_test.go | 82 +++++++++++ tools/gasbench/diff.go | 37 ++--- tools/gasbench/diff_test.go | 51 +++---- tools/gasbench/emit.go | 194 +++++++++++++++++------- tools/gasbench/emit_test.go | 179 ++++++++++++++++++---- tools/gasbench/run.sh | 7 + tools/gasbench/stats.go | 21 +-- 13 files changed, 799 insertions(+), 274 deletions(-) create mode 100644 tools/gasbench/crossrun.go create mode 100644 tools/gasbench/crossrun_test.go diff --git a/go.mod b/go.mod index 66bfbda919..89c52f5b74 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,7 @@ require ( golang.org/x/crypto v0.47.0 golang.org/x/exp v0.0.0-20260112195511-716be5621a96 golang.org/x/net v0.49.0 + golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5 golang.org/x/sync v0.19.0 golang.org/x/sys v0.40.0 golang.org/x/time v0.13.0 @@ -112,6 +113,7 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect github.com/cloudflare/circl v1.6.3 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect diff --git a/go.sum b/go.sum index 86a857402a..950c3ff285 100644 --- a/go.sum +++ b/go.sum @@ -659,6 +659,7 @@ github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkT github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= github.com/aclements/go-gg v0.0.0-20170118225347-6dbb4e4fefb0/go.mod h1:55qNq4vcpkIuHowELi5C8e+1yUHtoLoOUR9QU5j7Tes= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= @@ -2287,6 +2288,7 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5 h1:ObuXPmIgI4ZMyQLIz48cJYgSyWdjUXc2SZAdyJMwEAU= golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5/go.mod h1:UBKtEnL8aqnd+0JHqZ+2qoMDwtuy6cYhhKNoHLBiTQc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/tools/gasbench/AGENTS.md b/tools/gasbench/AGENTS.md index 68f517635f..a7a7d72c22 100644 --- a/tools/gasbench/AGENTS.md +++ b/tools/gasbench/AGENTS.md @@ -29,6 +29,13 @@ file is the short orientation. override — production never reprices a scalar opcode (stock `vm.Config{}`, no custom jump table); the only Sei gas param is `SeiSstoreSetGasEIP2200`, a storage opcode and out of scope here. +- **Data-dependent ops: verify the operands hit the real kernel.** For a + `DataDependent` spec, confirm `seedOperands` exercise the intended path, not + a `holiman/uint256` short-circuit — equal or degenerate operands make + `DIV(x,x)`/`MOD(x,x)` return without dividing, so the row would time a + compare. The gas self-check will NOT catch this (gas is unchanged); it's a + timing trap. `seedOperands` is ascending (dividend above divisor, smallest at + the modulus slot) for this reason — see README.md "Differential construction". - Keep code comments lean (what, not why); put methodology/rationale in README.md instead of growing doc comments. @@ -47,12 +54,13 @@ README.md "Running it" / "Output schema". | File | Contents | |---|---| | `gasbench.go` | timing core: `Measure`, `Config`, `Series`, rusage snapshot | -| `diff.go` | `Subtract`: baseline/target differencing, `Diff`, the acceptance gate | -| `stats.go` | `Summarize`: median/stddev/CoV/standard-error over a sample series | +| `diff.go` | `Subtract`: per-pass baseline/target differencing, `Diff` (medians/delta/gas/CoV) | +| `stats.go` | `Summarize`: median/stddev/CoV over a sample series | +| `crossrun.go` | `analyzeCrossRun`: cross-run benchmath verdict (paired delta CI gate + advisory Mann-Whitney p) | | `program.go` | `Program`: warmed tracer-free EVM environment, one bytecode input | | `programs.go` | `OpSpec`/`Specs`/`Case`/`BuildCaseWith`: the differential bytecode construction | -| `emit.go` | `Run`, CSV/NDJSON output | -| `emit_test.go` | pins `Run.Status` as a pure function of `Diff.Significant` | +| `emit.go` | `Run`, the verdict (`distinguishable`/`classifyStatus`), CSV/NDJSON output | +| `emit_test.go` | pins `Run.Status`/CI as pure functions of the `crossRun` verdict (`classifyStatus`, `distinguishable`) | | `bench_test.go` | `BenchmarkOpcodes`: wires the above into `go test -bench` | | `run.sh` | pinned-core runner + operator checklist for turbo/governor/isolation | | `README.md` | operator quickstart + full rationale: construction, acceptance gate, diagnostics | diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index d1c2c15281..c6bbece16c 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -2,11 +2,19 @@ Differential microbenchmark harness that measures per-opcode EVM execution time and correlates it with gas cost. Built to unblock the gas-repricing -project's F2 hypothesis test: does gas cost track execution time. - -Design: `designs/gas-repricing-telemetry/gas-vs-time-instrumentation.md` in -the `bdchatham-designs` repo. Noise-floor calibration behind the acceptance -gate below: `designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md` +project's F2 hypothesis test: does gas cost track execution time — the same +"gas is not a faithful meter of execution cost" question Broken Metre (Perez & +Livshits, NDSS 2020, arXiv 1909.07220) posed for mainnet, answered here +fork-natively. The method — isolate an opcode by repetition, then convert +runtime to gas — is the accepted state of the art (EIP-7904, Compute Gas Cost +Increase), whose anchor-rate / MGas·s⁻¹ vocabulary the repricing framing adopts. + +Design: `designs/gas-repricing-telemetry/gas-vs-time-instrumentation.md` +(harness) and `gasbench-benchmath-integration.md` (the cross-run statistics and +acceptance gate) in the `bdchatham-designs` repo; prior-art lineage in +`research/gasbench-prior-art-scan.md`. Noise-floor calibration behind the +acceptance gate below: +`designs/gas-repricing-telemetry/research/microbenchmark-noise-isolation-tradeoffs.md` in the same repo. ## Quickstart @@ -22,20 +30,20 @@ GASBENCH_OUT_CSV=gasbench.csv GASBENCH_OUT_NDJSON=gasbench.ndjson \ tools/gasbench/run.sh | tee gasbench.log ``` -This benchmarks every scalar opcode in `Specs` and writes one row per -opcode *per count* to the output files (`-count=10` by default, so 10 rows -per opcode — `-count` multiplies the subtests inside one benchmark -invocation, and the file is written once at the end with all of them). +This benchmarks every scalar opcode in `Specs` and writes ONE aggregate row +per opcode to the output files (`-count=10` by default: `-count` reruns each +subtest inside one benchmark invocation, and benchmath folds those passes into +the row's cross-run CI — the file is written once at the end). Relative output paths land in `tools/gasbench/` (the test binary's working directory), not where you invoked `run.sh` — pass absolute paths to put -them elsewhere. The `tee` is optional: the CSV already carries the -cross-count history step 4 needs; the captured stdout is the -`benchstat`-consumable form of the same data. Roughly ten minutes at the -defaults; set `GASBENCH_ITERS=2000 GASBENCH_COUNT=1` for a fast first look. -On a laptop or shared box, `run.sh` will warn that results are indicative: -expect cheap opcodes (`ADD`'s per-op cost is ~1ns, a whole-program delta -close to the noise) to come back `status=insignificant` there, and don't -read anything into them. +them elsewhere. The `tee` is optional: it captures the `go test` log (per-opcode +metrics, host-health warnings), not measurement data — the CSV/NDJSON aggregate +is the output of record. Roughly ten minutes at the defaults; set +`GASBENCH_ITERS=2000 GASBENCH_COUNT=1` for a fast first look (note `-count=1` is +`underpowered` by construction — no finite CI). On a laptop or shared box, +`run.sh` will warn that results are indicative: expect cheap opcodes (`ADD`'s +per-op cost is ~1 ns, a whole-program delta close to the noise) to come back +`insignificant` or `sub_threshold` there, and don't read anything into them. **2. For real numbers, use a quiet, dedicated Linux host:** bare metal (no co-tenants), fixed or pinned CPU frequency, one core reserved for the run @@ -57,40 +65,57 @@ target-minus-baseline delta, so setup/loop overhead has already cancelled so it is smaller (ADD→1, MULMOD→4) and arity-dependent. `gas_used` exists to drive the construction self-check; **`const_gas` is the denominator for any repricing statement.** -- **ns-per-gas** = `exec_time_ns / reps / const_gas` — the number the - hypothesis test is about: time per gas-the-chain-charges. If pricing tracked - execution time it would be roughly constant across opcodes; the spread IS the - mispricing signal. Do NOT divide by `gas_used` — that differential denominator - is deflated by an arity-correlated factor (up to 3x) and would report a spread - even for a perfectly-priced instruction set. Quote the spread only across - `significant=true` rows: the cheap ops (DUP1, ADD…) are dominated by the - op-minus-filler marginal and routinely come back `insignificant` on a normal - host — do not anchor the low end of the spread on one. Comparing rows within +- **ns-per-gas** = `exec_time_ns / reps / const_gas` — **divide by `const_gas` + (nominal), never `gas_used`**: the differential denominator is deflated by an + arity-correlated factor (up to 3x) and would report a spread even for a + perfectly-priced instruction set. This is the number the hypothesis test is + about: time per gas-the-chain-charges (EIP-7904 frames the same quantity as an + anchor rate — its 100 MGas/s ≈ 10 ns/gas). If pricing tracked execution time + it would be roughly constant across opcodes; the spread IS the mispricing + signal. Quote the spread + only across `status==ok` rows: the cheap ops (DUP1, ADD…) are dominated by the + op-minus-filler marginal and routinely come back `insignificant` or + `sub_threshold` on a normal host — do not anchor the low end of the spread on + one. Comparing rows within one run cancels the common clock-speed factor, so the spread is meaningful even though the absolute nanoseconds are host-specific — but microarchitectural ratios (adder vs shifter vs multiplier cost) still differ between hosts, which is why repricing-grade numbers come from the step-2 host, not a laptop. -**4. Only trust a row that earns it:** - -- `significant=false` (the row's `status` reads `insignificant` — same - condition, derived field): the delta didn't clear its own measurement - uncertainty — treat the time as indistinguishable from zero at this - precision. Raise `GASBENCH_ITERS`, or move to the quiet host; don't - average or rank insignificant rows. +**4. Only trust a row that earns it — filter on `status`:** + +- `status=ok` is the only correlation-eligible verdict: the paired delta CI + excludes zero (distinguishable) AND the per-op median delta clears the + effect-size floor (`GASBENCH_MIN_PEROP_NS`, default 1.0 ns). Sign-check + `exec_time_ns`: a negative value is a measured *speedup* (target faster than + baseline), still distinguishable — the gate is "measurably different," never + "more expensive." +- `status=sub_threshold`: distinguishable but below the effect floor — "not + measurably more expensive in absolute time at this precision." This is NOT + "correctly priced": a cheap op can still be mispriced. Do not feed + `sub_threshold` rows into the ns-per-gas correlation. +- `status=insignificant`: the delta CI straddles zero — indistinguishable from + no difference at this precision. Raise `GASBENCH_ITERS` or `-count`, or move + to the quiet host; don't average or rank insignificant rows. +- `status=underpowered`: too few counts for a finite CI (`-count<6` at the + default confidence — precisely: `<2` counts, or any count whose CI bound is + non-finite at the requested confidence; see "Output schema"); `ci_lo`/`ci_hi` + are null. Raise `GASBENCH_COUNT`. - `high_variance=true`: advisory — the run saw unusual dispersion (noisy - neighbor, throttling). It does not invalidate a significant result, and + neighbor, throttling). It does not invalidate an `ok` result, and `nivcsw` tells you where to look: nonzero means the scheduler preempted the process mid-window (blame the host); near-zero alongside the high CoV points at a non-scheduler tail inside the harness itself (see "Active-benchmarking diagnostics"). -- Rerun-to-rerun agreement is the real confidence check: `run.sh` defaults - to `-count=10`, so the CSV has 10 rows per opcode — group by `input_id` - and look at the spread of `exec_time_ns` across counts (the same data is - in `gasbench.log` in `benchstat`-consumable form). A per-op delta that is - stable across counts (and across invocations) is the result; on a - dedicated pinned host, expect low-single-digit-percent spread. See - "Running it" for what `-count` does and doesn't give you. +- The cross-count agreement check is now IN the row, not something you + reconstruct: `run.sh` defaults to `-count=10`, and benchmath folds those 10 + passes into the paired delta CI (`ci_lo`/`ci_hi`, at `confidence`). A tight CI + that excludes zero IS the stable-across-counts result; a wide one that + straddles zero is the noise verdict. There is no longer a per-count row to + eyeball or a `benchstat`-consumable stdout to pipe — the aggregate row is the + output of record. On a dedicated pinned host, expect the CI half-width to be a + low-single-digit percent of `exec_time_ns`. See "Running it" for what + `-count` does and doesn't give you. **Scope:** scalar constant-gas opcodes only (arithmetic/bitwise/comparison/ stack). Parametric opcodes (`KECCAK256`/`EXP`/copies), the state-touching @@ -214,23 +239,67 @@ I-cache/D-cache/branch predictor, catch a broken program early), then - Timing uses `time.Now`/`time.Since` (monotonic, immune to NTP steps) around the tracer-free `RunOnce` call. -## Acceptance gate: Significant, not CoV - -`Diff.Significant` (`diff.go`) — `|DeltaNs| > SigmaK * Uncertainty` (and -`Uncertainty > 0`; a series with fewer than 2 samples has nothing to -estimate variance from and must never read as significant), where -`Uncertainty` is the two series' median standard errors propagated in -quadrature — is the acceptance gate. A raw per-series CoV of several percent +## Acceptance gate: paired delta CI + effect floor, not CoV + +The verdict lives in `emit.go` (`distinguishable`/`effectSizePass`/ +`classifyStatus`), computed at the cross-run (`-count`) layer by +`benchmath.AssumeNothing` (`crossrun.go`). It has two independent halves; a row +is accepted (`status=ok`) only if BOTH clear. + +**1. Distinguishable — the statistical half.** Each `-count` pass measures +baseline then target back-to-back in one pinned, GC-off window, so the design +is *paired*: `delta_k = target_median_k − baseline_median_k`. The gate is the +order-statistic CI on that per-count delta series excluding zero — +`distinguishable := ci_lo > 0 || ci_hi < 0` (`benchmath.AssumeNothing.Summary` +over the deltas). It is symmetric: a CI wholly below zero (a measured speedup) +is just as distinguishable as one wholly above — the gate is "measurably +different," never "more expensive." + +Paired, not unpaired, is load-bearing. Across-count common-mode drift +(frequency wander, cache-state shift — exactly what `-count` exists to survive) +moves both series together, so the paired delta cancels it. An unpaired +two-sample test over the baseline-median series vs the target-median series +loses the signal under that drift: a true +3 ns/op with ±30 ns shared drift +makes the two median series overlap, while the paired deltas `[3,3,3,…]` are +unmistakable and their CI cleanly excludes zero. `benchmath` exposes no +paired/signed-rank test, so the CI on the delta series IS the paired primitive. +The unpaired Mann-Whitney U p (`p_value`, over the baseline vs target median +series) is emitted as an **advisory column only** for cross-checking — NOT the +gate, so its known weaknesses (tie fragility at integer-ns medians, power loss +under shared drift) never touch the acceptance decision. + +**2. Effect-size floor — the practical half.** Distinguishable at large n is +not the same as meaningful. `status=ok` additionally requires the per-op median +delta (`exec_time_ns / reps`) to clear `GASBENCH_MIN_PEROP_NS` (default 1.0 ns, +compared in absolute value so a speedup still counts). A row that is +distinguishable but below the floor is `sub_threshold`, NOT `ok`. + +`sub_threshold` means "not measurably more expensive in absolute time at this +precision" — it does NOT mean "correctly priced." A cheap-but-mispriced op can +be demoted on the absolute-ns floor even though its ns-per-gas is an outlier; +the floor is a *measurability* guard, not a pricing verdict. **Only `status=ok` +rows are eligible for the ns-per-gas repricing correlation.** The 1.0 ns default +is a consumer-tunable calibration (it may demote ADD/DUP-class boundary ops on +some hosts — cite the noise-isolation research when retuning); set +`GASBENCH_MIN_PEROP_NS<=0` to disable the floor entirely. A gas-relative floor +(ns-per-nominal-gas deviation) is deferred pending the reference-anchor choice +(EIP-7904's 100 MGas/s is the citable candidate but is host-frequency-specific). + +(Distinguishability is decided at the cross-run n=10 layer, not the within-run +n≈20000 layer — at within-run n the test is over-powered, any sub-nanosecond +shift clears it; the effect floor is the second half of the gate.) + +**CoV is advisory, never the gate.** A raw per-series CoV of several percent under plain core-pinning (no kernel-level isolcpus/nohz_full/rcu_nocbs) is expected physics (the periodic scheduler tick, device IRQs, shared-cache contention that pinning alone can't remove), not a defect. No major benchmarking harness (JMH, criterion.rs, Go's own `benchstat`) gates on a fixed dispersion threshold either — all three gate on effect size against -the estimate's own uncertainty, same as `Significant`. +the estimate's own uncertainty. `Diff.HighVariance` (CoV above `CoVCeiling`, default 0.25) is advisory only: it flags a run worth a closer look (a noisy neighbor, a throttling event) but -never overrides `Significant`. The 0.25 default sits above the 4-8% CoV +never overrides `status`. The 0.25 default sits above the 4-8% CoV measured as normal on a dedicated, pinned, bare-metal host with no kernel-level isolation, and well below the ~40%+ territory a genuinely pathological run would show. See the research doc linked above for the full @@ -273,36 +342,78 @@ go test ./tools/gasbench/ -run '^$' -bench '^BenchmarkOpcodes$' \ `-benchtime=1x` matters: `BenchmarkOpcodes`'s inner timing loop is `Measure`'s, not `b.N`'s, so `b.N` must stay at 1. `-count=K` reruns each opcode subtest K times **within the same OS process** (inside a single -`BenchmarkOpcodes` invocation, so the output files accumulate all K rows -per opcode), not as K independent process forks — it gives -`benchstat`-style cross-run variance, not process-level independence. True process-level independence would need K -separate `go test` invocations. See the research doc for why this matters -less than it sounds: JMH's fork/warmup/measurement-iteration model is the -gold standard, but this MVP's within-process reruns already surfaced -reproducible per-opcode deltas (see the research doc's empirical run). - -Env vars: `GASBENCH_WARMUP`, `GASBENCH_ITERS` (override `DefaultConfig`), -`GASBENCH_SIGMA_K` (default 3), `GASBENCH_COV_CEILING` (default 0.25), -`GASBENCH_COUNT` (default 10), `GASBENCH_OUT_CSV`, `GASBENCH_OUT_NDJSON`. +`BenchmarkOpcodes` invocation); benchmath folds those K passes into the one +aggregate row's cross-run CI. This is cross-run variance, not process-level +independence — true process-level independence would need K separate `go test` +invocations. See the research doc for why this matters less than it sounds: +JMH's fork/warmup/measurement-iteration model is the gold standard, but this +MVP's within-process reruns already surfaced reproducible per-opcode deltas +(see the research doc's empirical run). **`-count>=6` is needed for a finite CI** +at the default confidence; below that a row reads `underpowered` with a +benchmath warning, so the default `-count=10` has headroom. + +Env vars: + +- `GASBENCH_WARMUP`, `GASBENCH_ITERS` — override `DefaultConfig`'s within-run + warmup / timed iterations. +- `GASBENCH_COUNT` (default 10) — cross-run `-count` passes; see the finite-CI + minimum above. +- `GASBENCH_ALPHA` (default 0.05) — `CompareAlpha` behind the advisory + `p_value` (Mann-Whitney U); does not affect the gate. +- `GASBENCH_CI_CONFIDENCE` (default 0.95) — confidence for the paired delta CI + (the gate). Raising it raises the minimum `-count` for a finite CI (~6 at + 0.95), so bump `GASBENCH_COUNT` in lockstep. +- `GASBENCH_MIN_PEROP_NS` (default 1.0) — effect-size floor on `|per-op median + delta ns|`; distinguishable-but-below → `sub_threshold`. Set `<=0` to disable. +- `GASBENCH_COV_CEILING` (default 0.25) — advisory `high_variance` ceiling on + per-series CoV; never gates the verdict. +- `GASBENCH_OUT_CSV`, `GASBENCH_OUT_NDJSON` — output paths. ## Output schema -One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON: +One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON. Columns are in +`csvHeader` order: -| Field | Meaning | +| Column | Meaning | |---|---| | `input_id` | opcode id, e.g. `ADD` | | `class` | opcode family, e.g. `arithmetic` | | `reps` | opcode executions the delta represents | -| `gas_used` | *differential* whole-program gas delta (target - baseline), net of filler; per-op = `gas_used/reps`. Drives the construction self-check — NOT the repricing denominator | +| `gas_used` | *differential* whole-program gas delta (target − baseline), net of filler; per-op = `gas_used/reps`. Drives the construction self-check — NOT what the chain charges, NOT the repricing denominator | | `const_gas` | nominal per-op gas the chain charges (jump-table `ConstGas`); the denominator for ns-per-gas — see "Read the output" | -| `exec_time_ns` | whole-program time delta, ns; per-op = `exec_time_ns/reps` | -| `status` | `ok` if `significant`, else `insignificant` — never gated on CoV | -| `iterations` | timed iterations behind each series | -| `cov` | worse (max) of the baseline/target series CoV — advisory only | -| `significant` | the acceptance gate | -| `high_variance` | advisory: CoV exceeded `CoVCeiling` | -| `nvcsw` / `nivcsw` | worse (max) of the baseline/target voluntary/involuntary context-switch counts | +| `exec_time_ns` | **median** whole-program time delta over the `-count` passes (`Summary.Center`), ns; always finite; per-op = `exec_time_ns/reps`. **Sign-check it: a negative value is a measured speedup.** | +| `count` | cross-run n: number of `-count` passes aggregated | +| `iterations` | within-run timed iterations behind each series | +| `ci_lo` | order-statistic CI low on `exec_time_ns` (whole-program ns). **Nullable**: empty CSV field / JSON `null` when the bound is non-finite (underpowered) | +| `ci_hi` | CI high; same nullable rule | +| `confidence` | actual CI confidence (≥ requested) | +| `distinguishable` | **the gate, statistical half:** paired delta CI excludes zero (`ci_lo>0 \|\| ci_hi<0`), computed from the raw benchmath bounds — on an underpowered row the emitted `ci_lo`/`ci_hi` are null and this reads `false` | +| `effect_size_pass` | the gate, practical half: per-op median delta clears `GASBENCH_MIN_PEROP_NS` | +| `p_value` | **advisory only:** unpaired Mann-Whitney U p over baseline vs target medians; NOT the gate | +| `alpha` | `CompareAlpha` behind the advisory `p_value` | +| `status` | `ok \| sub_threshold \| insignificant \| underpowered \| error` — see below; never gated on CoV | +| `cov` | worse (max) of the baseline/target series CoV, max across counts — advisory only | +| `high_variance` | advisory: CoV exceeded `CoVCeiling` on any count | +| `nvcsw` / `nivcsw` | worse (max) of the baseline/target voluntary/involuntary context-switch counts, max across counts | + +`status` enum: + +- `ok` — distinguishable AND clears the effect floor. **The only + correlation-eligible verdict.** +- `sub_threshold` — distinguishable but below the effect floor: "not measurably + more expensive in absolute time," NOT "correctly priced" (a cheap op can + still be mispriced). +- `insignificant` — the delta CI straddles zero: not distinguishable at this + precision. +- `underpowered` — too few counts for a finite CI (`count<2`, or a non-finite + bound at the requested confidence); `ci_lo`/`ci_hi` are null. +- `error` — reserved for a per-case measurement failure; not currently emitted + (an invalid program fails the benchmark loudly instead). + +**Consumer note:** filter on `status` (only `ok` is correlation-eligible) and +sign-check `exec_time_ns` (a negative value = a measured speedup). `ci_lo`/`ci_hi` +are null on underpowered rows — an older CSV wrote `+Inf` here, the current CSV +writes an empty field. A per-`Case` self-check (`bench_test.go`) also verifies the measured gas delta matches the definitional expected delta from `BuildCaseWith`'s algebra diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index 84e3f745ea..22bce1ad9d 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -13,17 +13,24 @@ func BenchmarkOpcodes(b *testing.B) { cases := BuildCases() def := DefaultConfig() cfg := Config{ - Warmup: envInt("GASBENCH_WARMUP", def.Warmup), - Iterations: envInt("GASBENCH_ITERS", def.Iterations), + Warmup: envInt(b, "GASBENCH_WARMUP", def.Warmup), + Iterations: envInt(b, "GASBENCH_ITERS", def.Iterations), DisableGC: def.DisableGC, LockThread: def.LockThread, } - sigmaK := envFloat("GASBENCH_SIGMA_K", 3) // 0.25 default: see README.md "Acceptance gate" for the calibration. - covCeiling := envFloat("GASBENCH_COV_CEILING", 0.25) + covCeiling := envFloat(b, "GASBENCH_COV_CEILING", 0.25) + alpha := envFloat(b, "GASBENCH_ALPHA", 0.05) + confidence := envFloat(b, "GASBENCH_CI_CONFIDENCE", 0.95) + // Effect-size floor: a per-op median delta below this (ns) is distinguishable + // but not "meaningfully more expensive" -> sub_threshold. See README "Acceptance gate". + minPerOpNs := envFloat(b, "GASBENCH_MIN_PEROP_NS", 1.0) - var runs []Run - for _, c := range cases { + // One accumulator per case, indexed by the case-loop index. Go 1.22+ + // per-iteration loop-var capture makes &collectors[i] inside the closure + // bind to cases[i]'s slot regardless of how -count reruns interleave. + collectors := make([]opcodeAccum, len(cases)) + for i, c := range cases { b.Run(c.OpcodeID, func(b *testing.B) { baseProg, err := NewProgram(c.Baseline) if err != nil { @@ -43,7 +50,7 @@ func BenchmarkOpcodes(b *testing.B) { b.Fatal(err) } - d := Subtract(c.OpcodeID, base, tgt, c.Reps, sigmaK, covCeiling) + d := Subtract(c.OpcodeID, base, tgt, c.Reps, covCeiling) // Self-check of the harness construction (see README.md "Output // schema"), not of opcode timing. Insensitive to a @@ -54,68 +61,141 @@ func BenchmarkOpcodes(b *testing.B) { c.OpcodeID, d.GasDelta, c.ExpectedGasDelta) } + // Lock-free like sink (gasbench.go): subtests and -count reruns are + // sequential (no b.Parallel). A future parallelize needs a mutex. + acc := &collectors[i] + acc.iterations = cfg.Iterations + // The gas delta is definitional and must be identical every count; + // a drift means the differential construction is not deterministic. + if acc.gasDeltaSet && d.GasDelta != acc.gasDelta { + b.Errorf("%s: gas delta not identical across counts: %d != first-count %d", + c.OpcodeID, d.GasDelta, acc.gasDelta) + } + if !acc.gasDeltaSet { + acc.gasDelta, acc.gasDeltaSet = d.GasDelta, true + } + acc.baseMedians = append(acc.baseMedians, d.BaselineMedian) + acc.tgtMedians = append(acc.tgtMedians, d.TargetMedian) + acc.deltas = append(acc.deltas, d.DeltaNs) + acc.covMax = max(acc.covMax, d.BaselineCoV, d.TargetCoV) + acc.highVariance = acc.highVariance || d.HighVariance + acc.nvcsw = max(acc.nvcsw, d.BaselineNvcsw, d.TargetNvcsw) + acc.nivcsw = max(acc.nivcsw, d.BaselineNivcsw, d.TargetNivcsw) + b.ReportMetric(d.BaselineMedian, "baseline-median-ns") b.ReportMetric(d.TargetMedian, "target-median-ns") b.ReportMetric(d.PerOpNs, "per-op-ns") - b.ReportMetric(d.PerOpGas, "per-op-gas") + // per-op-gas-delta is differential (net of filler); const-gas is the + // nominal denominator for ns-per-gas. + b.ReportMetric(d.PerOpGas, "per-op-gas-delta") + b.ReportMetric(float64(c.ConstGas), "const-gas") for _, w := range append(base.Warnings, tgt.Warnings...) { b.Logf("%s: %s", c.OpcodeID, w) } - // Significant is the acceptance gate; HighVariance is advisory - // only and never overrides it (see emit.go, diff.go doc comments). - if !d.Significant { - b.Logf("%s: delta %.1fns within noise (uncertainty %.1fns, %gσ) -- marginal cost not distinguishable from zero at this precision", - c.OpcodeID, d.DeltaNs, d.Uncertainty, sigmaK) - } if d.HighVariance { - b.Logf("%s: series CoV above health-check ceiling %.4g (baseline=%.4g target=%.4g, nivcsw base=%d tgt=%d) -- worth investigating the host, does not invalidate a significant result", + b.Logf("%s: series CoV above health-check ceiling %.4g (baseline=%.4g target=%.4g, nivcsw base=%d tgt=%d) -- advisory only, does not gate the verdict", c.OpcodeID, covCeiling, d.BaselineCoV, d.TargetCoV, base.NivcswDelta, tgt.NivcswDelta) } - - runs = append(runs, NewRun(c, d, cfg.Iterations)) }) } + + // Aggregate once, in Specs order, after all counts have accumulated. + runs := make([]Run, 0, len(cases)) + for i := range cases { + runs = append(runs, aggregate(b, &collectors[i], cases[i], alpha, confidence, minPerOpNs)) + } writeRuns(b, runs) } +// opcodeAccum accumulates one opcode's per-count raw data across the -count +// reruns of its subtest. Test-local: production analyzeCrossRun takes +// crossRunInput instead. The Case is not stored here -- aggregation reads it +// from cases[i]. +type opcodeAccum struct { + iterations int + baseMedians []float64 // within-run median per count, ns + tgtMedians []float64 // within-run median per count, ns + deltas []float64 // per-count delta, ns (tgtMed - baseMed) + gasDelta uint64 // exact; guarded identical every count + gasDeltaSet bool + covMax float64 // running max of per-count worse-of-pair CoV (advisory) + highVariance bool // OR across counts (advisory) + nvcsw int64 // running max of worse-of-pair voluntary ctx switches + nivcsw int64 // running max of worse-of-pair involuntary ctx switches +} + +// aggregate builds the cross-run input from the collector, runs the benchmath +// analysis, and composes the per-opcode Run. Thin glue: the verdict lives in +// NewRun/classifyStatus (emit.go). +func aggregate(b *testing.B, acc *opcodeAccum, c Case, alpha, confidence, minPerOpNs float64) Run { + cr := analyzeCrossRun(crossRunInput{ + BaselineMedians: acc.baseMedians, + TargetMedians: acc.tgtMedians, + Deltas: acc.deltas, + }, alpha, confidence) + // Surface benchmath's cross-run warnings (e.g. too few counts for a finite + // CI, or n below the U-test's minimum) so an underpowered/degenerate run is + // visible, not silent. + for _, w := range cr.Warnings { + b.Logf("%s: %v", c.OpcodeID, w) + } + return NewRun(c, cr, acc.gasDelta, len(acc.deltas), acc.iterations, hostHealth{ + CoV: acc.covMax, + HighVariance: acc.highVariance, + Nvcsw: acc.nvcsw, + Nivcsw: acc.nivcsw, + }, minPerOpNs) +} + func writeRuns(b *testing.B, runs []Run) { if p := os.Getenv("GASBENCH_OUT_CSV"); p != "" { - f, err := os.Create(p) - if err != nil { - b.Fatalf("create csv: %v", err) - } - defer f.Close() - if err := WriteCSV(f, runs); err != nil { - b.Fatalf("write csv: %v", err) - } + writeFile(b, p, "csv", func(f *os.File) error { return WriteCSV(f, runs) }) } if p := os.Getenv("GASBENCH_OUT_NDJSON"); p != "" { - f, err := os.Create(p) - if err != nil { - b.Fatalf("create ndjson: %v", err) - } - defer f.Close() - if err := WriteNDJSON(f, runs); err != nil { - b.Fatalf("write ndjson: %v", err) - } + writeFile(b, p, "ndjson", func(f *os.File) error { return WriteNDJSON(f, runs) }) } } -func envInt(key string, def int) int { - if v := os.Getenv(key); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } +// writeFile checks the Close error too: a failed final flush silently truncates +// the results file the operator then trusts as measurement data. +func writeFile(b *testing.B, path, kind string, write func(*os.File) error) { + f, err := os.Create(path) + if err != nil { + b.Fatalf("create %s: %v", kind, err) + } + if err := write(f); err != nil { + f.Close() + b.Fatalf("write %s: %v", kind, err) + } + if err := f.Close(); err != nil { + b.Fatalf("close %s: %v", kind, err) } - return def } -func envFloat(key string, def float64) float64 { - if v := os.Getenv(key); v != "" { - if f, err := strconv.ParseFloat(v, 64); err == nil { - return f - } +// envInt/envFloat fail loud on a set-but-unparseable value rather than silently +// falling back to the default: a typo like GASBENCH_ITERS=2O00 should stop the +// run, not quietly measure something the operator didn't ask for. +func envInt(b *testing.B, key string, def int) int { + v := os.Getenv(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil { + b.Fatalf("%s=%q: not an integer: %v", key, v, err) + } + return n +} + +func envFloat(b *testing.B, key string, def float64) float64 { + v := os.Getenv(key) + if v == "" { + return def + } + f, err := strconv.ParseFloat(v, 64) + if err != nil { + b.Fatalf("%s=%q: not a float: %v", key, v, err) } - return def + return f } diff --git a/tools/gasbench/crossrun.go b/tools/gasbench/crossrun.go new file mode 100644 index 0000000000..67900172e6 --- /dev/null +++ b/tools/gasbench/crossrun.go @@ -0,0 +1,59 @@ +package gasbench + +import ( + "math" + "slices" + + "golang.org/x/perf/benchmath" +) + +// crossRunInput is one opcode's cross-run series: one entry per -count pass. +// Named fields prevent transposing the three same-typed slices at the call +// site. Deltas[k] = TargetMedians[k] - BaselineMedians[k]. +type crossRunInput struct { + BaselineMedians []float64 // within-run median per count, ns + TargetMedians []float64 // within-run median per count, ns + Deltas []float64 // per-count delta, ns +} + +// crossRun is the benchmath verdict for one opcode's cross-run series. +// +// CILo/CIHi are the raw order-statistic CI bounds on the delta and may be +// non-finite (too few counts). The acceptance gate reads them raw (a ±Inf +// bound is simply not > 0 / < 0, so an underpowered row never reads +// distinguishable); the emit layer maps non-finite to a null column. +type crossRun struct { + MedianDeltaNs float64 // Summary.Center -- median whole-program delta + CILo, CIHi float64 // Summary.Lo/Hi -- order-statistic CI; ±Inf when underpowered + Confidence float64 // Summary.Confidence -- actual (>= requested) + P float64 // Compare.P -- advisory only, never the gate + Alpha float64 // Compare.Alpha + Underpowered bool // < 2 counts, or a non-finite CI bound + Warnings []error // Summary.Warnings ++ Compare.Warnings +} + +// analyzeCrossRun runs the paired delta CI (the acceptance gate, via Summary +// over the per-count deltas) and the advisory unpaired Mann-Whitney p (via +// Compare over the baseline vs target median series). The paired CI is the +// primitive that survives common-run drift; the p-value is surfaced for +// cross-checking only. Inputs are cloned: NewSample sorts in place. +func analyzeCrossRun(in crossRunInput, alpha, confidence float64) crossRun { + thr := &benchmath.Thresholds{CompareAlpha: alpha} + + sum := benchmath.AssumeNothing.Summary(benchmath.NewSample(slices.Clone(in.Deltas), thr), confidence) + cmp := benchmath.AssumeNothing.Compare( + benchmath.NewSample(slices.Clone(in.BaselineMedians), thr), + benchmath.NewSample(slices.Clone(in.TargetMedians), thr), + ) + + return crossRun{ + MedianDeltaNs: sum.Center, + CILo: sum.Lo, + CIHi: sum.Hi, + Confidence: sum.Confidence, + P: cmp.P, + Alpha: cmp.Alpha, + Underpowered: len(in.Deltas) < 2 || math.IsInf(sum.Lo, 0) || math.IsInf(sum.Hi, 0), + Warnings: slices.Concat(sum.Warnings, cmp.Warnings), + } +} diff --git a/tools/gasbench/crossrun_test.go b/tools/gasbench/crossrun_test.go new file mode 100644 index 0000000000..6ce54d0eb8 --- /dev/null +++ b/tools/gasbench/crossrun_test.go @@ -0,0 +1,82 @@ +package gasbench + +import ( + "math" + "testing" +) + +// TestAnalyzeCrossRunPairedSurvivesDrift is the F1 scenario: a true +3ns/op shift +// under common-mode drift makes the baseline/target median series overlap, so the +// advisory unpaired p is WEAK (fails to distinguish), yet the paired delta CI +// cleanly excludes zero. This is the reason the gate reads the paired CI, not +// Compare -- so the test asserts both sides of that split. +func TestAnalyzeCrossRunPairedSurvivesDrift(t *testing.T) { + base := []float64{200, 215, 208, 230, 205, 220, 212, 218, 209, 225} + tgt := make([]float64, len(base)) + deltas := make([]float64, len(base)) + for i, b := range base { + tgt[i] = b + 3 + deltas[i] = tgt[i] - b + } + cr := analyzeCrossRun(crossRunInput{BaselineMedians: base, TargetMedians: tgt, Deltas: deltas}, 0.05, 0.95) + + if cr.Underpowered { + t.Fatalf("n=10 must not be underpowered") + } + if math.IsInf(cr.CILo, 0) || math.IsInf(cr.CIHi, 0) { + t.Fatalf("n=10 CI must be finite, got [%v, %v]", cr.CILo, cr.CIHi) + } + // The advisory unpaired test must FAIL to distinguish under drift -- the whole + // point of gating on the paired CI instead. + if cr.P <= cr.Alpha { + t.Errorf("unpaired p should be weak under drift (> alpha), got P=%v alpha=%v", cr.P, cr.Alpha) + } + // The paired CI must exclude zero. + if cr.CILo <= 0 { + t.Errorf("paired CI must exclude zero (Lo>0), got Lo=%v", cr.CILo) + } + if cr.MedianDeltaNs != 3 { + t.Errorf("median delta got %v, want 3", cr.MedianDeltaNs) + } +} + +// TestAnalyzeCrossRunStraddlesZero pins the finite-CI-includes-zero path: a +// zero-centered delta series is measurable (not underpowered) but the CI +// straddles zero, so the gate must NOT read it as distinguishable. +func TestAnalyzeCrossRunStraddlesZero(t *testing.T) { + deltas := []float64{-4, -3, -1, 0, 1, 1, 2, 3, 4, 5} + base := make([]float64, len(deltas)) + tgt := make([]float64, len(deltas)) + for i, d := range deltas { + base[i] = 100 + tgt[i] = 100 + d + } + cr := analyzeCrossRun(crossRunInput{BaselineMedians: base, TargetMedians: tgt, Deltas: deltas}, 0.05, 0.95) + + if cr.Underpowered { + t.Fatalf("n=10 must not be underpowered") + } + if math.IsInf(cr.CILo, 0) || math.IsInf(cr.CIHi, 0) { + t.Fatalf("n=10 CI must be finite, got [%v, %v]", cr.CILo, cr.CIHi) + } + if !(cr.CILo <= 0 && cr.CIHi >= 0) { + t.Errorf("zero-centered deltas: CI must straddle zero, got [%v, %v]", cr.CILo, cr.CIHi) + } +} + +// TestAnalyzeCrossRunUnderpowered pins the degenerate single-count path: a finite +// point median but a non-finite CI, flagged underpowered, no panic. +func TestAnalyzeCrossRunUnderpowered(t *testing.T) { + cr := analyzeCrossRun(crossRunInput{ + BaselineMedians: []float64{200}, + TargetMedians: []float64{203}, + Deltas: []float64{3}, + }, 0.05, 0.95) + + if !cr.Underpowered { + t.Fatalf("n=1 must be underpowered") + } + if cr.MedianDeltaNs != 3 { + t.Errorf("point median of a single delta got %v, want 3", cr.MedianDeltaNs) + } +} diff --git a/tools/gasbench/diff.go b/tools/gasbench/diff.go index c305115bfd..e5e9f269f6 100644 --- a/tools/gasbench/diff.go +++ b/tools/gasbench/diff.go @@ -1,11 +1,9 @@ package gasbench -import "math" - // Diff is the difference-of-medians subtraction of a baseline series from a -// target series, with the median's standard error propagated in quadrature -// as Uncertainty. See README.md for the estimator choice and the -// Significant-not-CoV acceptance-gate rationale. +// target series for one -count pass. Distinguishability is decided at the +// cross-run layer (crossrun.go), not here: Diff carries only the per-pass +// point medians, delta, gas, and advisory health signals. type Diff struct { OpcodeID string @@ -14,13 +12,10 @@ type Diff struct { BaselineCoV float64 TargetCoV float64 - DeltaNs float64 // per-program time attributable to the opcode reps - Uncertainty float64 // 1-sigma, propagated in quadrature (ns) - SigmaK float64 // significance multiple applied - Significant bool // |DeltaNs| > SigmaK*Uncertainty -- the acceptance gate (see emit.go's Status) + DeltaNs float64 // per-program time attributable to the opcode reps - // HighVariance is advisory only (CoV exceeded CoVCeiling); it never gates - // Significant. See README.md "Acceptance gate". + // HighVariance is advisory only (CoV exceeded CoVCeiling). See README.md + // "Acceptance gate". HighVariance bool CoVCeiling float64 @@ -40,14 +35,11 @@ type Diff struct { PerOpGas float64 // GasDelta / Reps } -// Subtract computes the opcode cost from baseline/target series. sigmaK sets -// the significance band (e.g. 3 ~ 99.7% for a normal center) and is the -// acceptance gate. covCeiling is an advisory health-check ceiling, not a -// gate: a raw per-series CoV above it flags the run for a closer look -// (a possible neighbor or throttling event) without invalidating an -// otherwise-significant result. reps normalizes the per-program delta to one -// opcode. -func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covCeiling float64) Diff { +// Subtract computes the per-pass opcode cost from baseline/target series. +// covCeiling is an advisory health-check ceiling, not a gate: a raw per-series +// CoV above it flags the run for a closer look (a possible neighbor or +// throttling event). reps normalizes the per-program delta to one opcode. +func Subtract(opcodeID string, baseline, target Series, reps int, covCeiling float64) Diff { bs := Summarize(baseline.Samples) ts := Summarize(target.Samples) @@ -57,7 +49,6 @@ func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covCei TargetMedian: ts.Median, BaselineCoV: bs.CoV, TargetCoV: ts.CoV, - SigmaK: sigmaK, CoVCeiling: covCeiling, BaselineNvcsw: baseline.NvcswDelta, TargetNvcsw: target.NvcswDelta, @@ -66,12 +57,6 @@ func Subtract(opcodeID string, baseline, target Series, reps int, sigmaK, covCei Reps: reps, } d.DeltaNs = ts.Median - bs.Median - d.Uncertainty = math.Hypot(ts.SEMedian, bs.SEMedian) - // Uncertainty == 0 (e.g. Iterations < 2, or an improbable zero-variance - // sample) means no variance was ever estimated, not that the delta is - // perfectly known -- Significant must never be true from an - // unestimated Uncertainty. - d.Significant = d.Uncertainty > 0 && math.Abs(d.DeltaNs) > sigmaK*d.Uncertainty d.HighVariance = bs.CoV > covCeiling || ts.CoV > covCeiling if target.GasUsed >= baseline.GasUsed { diff --git a/tools/gasbench/diff_test.go b/tools/gasbench/diff_test.go index 28d7b26716..bc74cd2873 100644 --- a/tools/gasbench/diff_test.go +++ b/tools/gasbench/diff_test.go @@ -5,37 +5,28 @@ import ( "time" ) -// TestSubtractNeverSignificantWithZeroUncertainty pins that a series with an -// unestimated variance (fewer than 2 samples, or an improbable zero-variance -// sample) never yields Significant=true, however large the delta -- a raw -// |DeltaNs| > 0 must not stand in for a real effect-size check against an -// uncertainty that was never estimated. -func TestSubtractNeverSignificantWithZeroUncertainty(t *testing.T) { - low1 := Series{Samples: []time.Duration{100 * time.Nanosecond}} - high1 := Series{Samples: []time.Duration{100000 * time.Nanosecond}} - low2 := Series{Samples: []time.Duration{100 * time.Nanosecond, 100 * time.Nanosecond}} - high2 := Series{Samples: []time.Duration{100000 * time.Nanosecond, 100000 * time.Nanosecond}} - - cases := []struct { - name string - baseline Series - target Series - }{ - {"single-sample baseline, zero-variance target", low1, high2}, - {"zero-variance baseline, single-sample target", low2, high1}, - {"both single-sample", low1, high1}, - {"zero-variance both", low2, high2}, +// TestSubtractPerPass pins the slimmed per-pass contract: Subtract carries the +// point median delta and the exact gas delta, with no significance verdict +// (distinguishability now lives at the cross-run layer, crossrun.go). +func TestSubtractPerPass(t *testing.T) { + baseline := Series{ + GasUsed: 100, + Samples: []time.Duration{100 * time.Nanosecond, 102 * time.Nanosecond, 104 * time.Nanosecond}, + } + target := Series{ + GasUsed: 130, + Samples: []time.Duration{110 * time.Nanosecond, 112 * time.Nanosecond, 114 * time.Nanosecond}, } - for _, tc := range cases { - d := Subtract("ADD", tc.baseline, tc.target, 1, 3, 0.25) - if d.Uncertainty != 0 { - t.Errorf("%s: Uncertainty = %v, want 0 for this fixture", tc.name, d.Uncertainty) - continue - } - if d.Significant { - t.Errorf("%s: Significant = true with Uncertainty == 0 and DeltaNs = %v -- an unestimated uncertainty must never pass the gate", - tc.name, d.DeltaNs) - } + d := Subtract("ADD", baseline, target, 10, 0.25) + + if d.DeltaNs != 10 { + t.Errorf("DeltaNs = %v, want 10 (target median - baseline median)", d.DeltaNs) + } + if d.GasDelta != 30 { + t.Errorf("GasDelta = %d, want 30", d.GasDelta) + } + if d.PerOpNs != 1 { + t.Errorf("PerOpNs = %v, want 1 (DeltaNs/reps)", d.PerOpNs) } } diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index e50c74d326..3b775f3dae 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -5,78 +5,151 @@ import ( "encoding/json" "fmt" "io" + "math" "strconv" ) -// Run is one emitted measurement row: the per-opcode differential result, -// ready for a gas-vs-time correlation. See README.md "Output schema". +// Run is one emitted measurement row: the per-opcode aggregate over all +// -count passes, ready for a gas-vs-time correlation. See README.md "Output +// schema". type Run struct { - InputID string `json:"input_id"` // opcode id, e.g. "ADD" - Class string `json:"class"` // opcode family, e.g. "arithmetic" - Reps int `json:"reps"` // opcode executions the delta represents - GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps -- this is the DIFFERENTIAL (net of filler), not the gas the chain charges - ConstGas uint64 `json:"const_gas"` // nominal per-op gas the chain charges (jump-table ConstGas); the repricing-relevant denominator for ns-per-gas. See README.md "Read the output" - ExecTimeNs float64 `json:"exec_time_ns"` // whole-program time delta, ns (target median - baseline median); per-op = ExecTimeNs/Reps - Status string `json:"status"` // ok if Significant, else insignificant -- never gated on CoV - Iterations int `json:"iterations"` // timed iterations behind each series - CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV -- advisory only - Significant bool `json:"significant"` // |delta| exceeds SigmaK * propagated uncertainty -- the acceptance gate - HighVariance bool `json:"high_variance"` // advisory: CoV exceeded the health-check ceiling; does not invalidate Status=ok - Nvcsw int64 `json:"nvcsw"` // worse (max) of the baseline/target voluntary-context-switch count; see README.md "Active-benchmarking diagnostics" for interpreting a zero here - Nivcsw int64 `json:"nivcsw"` // worse (max) of the baseline/target involuntary-context-switch count; see README.md "Active-benchmarking diagnostics" for interpreting a zero here + InputID string `json:"input_id"` // opcode id, e.g. "ADD" + Class string `json:"class"` // opcode family, e.g. "arithmetic" + Reps int `json:"reps"` // opcode executions the delta represents + GasUsed uint64 `json:"gas_used"` // whole-program gas delta (target - baseline); per-op = GasUsed/Reps -- this is the DIFFERENTIAL (net of filler), not the gas the chain charges + ConstGas uint64 `json:"const_gas"` // nominal per-op gas the chain charges (jump-table ConstGas); the repricing-relevant denominator for ns-per-gas. See README.md "Read the output" + ExecTimeNs float64 `json:"exec_time_ns"` // median whole-program delta over the counts (Summary.Center); always finite; per-op = ExecTimeNs/Reps + Count int `json:"count"` // cross-run n: number of -count passes aggregated + Iterations int `json:"iterations"` // timed iterations behind each series + + // CILo/CIHi are the order-statistic CI bounds on ExecTimeNs (whole-program + // ns). Nullable: nil (JSON null / CSV empty) when the raw benchmath bound + // is non-finite (underpowered). THE GATE (Distinguishable) reads the raw + // bounds, never these pointers -- see crossrun.go and distinguishable. + CILo *float64 `json:"ci_lo"` + CIHi *float64 `json:"ci_hi"` + Confidence float64 `json:"confidence"` // actual CI confidence (>= requested) + Distinguishable bool `json:"distinguishable"` // paired delta CI excludes zero (ci_lo>0 || ci_hi<0) -- the statistical half of the gate (F1) + EffectSizePass bool `json:"effect_size_pass"` // per-op median delta clears the effect floor -- the practical half of the gate + PValue float64 `json:"p_value"` // advisory only: unpaired Mann-Whitney U p over baseline vs target medians; NOT the gate + Alpha float64 `json:"alpha"` // CompareAlpha behind the advisory p_value + Status string `json:"status"` // ok | sub_threshold | insignificant | underpowered | error -- never gated on CoV + + CoV float64 `json:"cov"` // worse (max) of the baseline/target series CoV, max across counts -- advisory only + HighVariance bool `json:"high_variance"` // advisory: CoV exceeded the health-check ceiling on any count; does not invalidate Status=ok + Nvcsw int64 `json:"nvcsw"` // worse (max) of the baseline/target voluntary-context-switch count, max across counts; see README.md "Active-benchmarking diagnostics" + Nivcsw int64 `json:"nivcsw"` // worse (max) of the baseline/target involuntary-context-switch count, max across counts; see README.md "Active-benchmarking diagnostics" } -// Status values. See README.md "Acceptance gate" for why Status is -// Significant-driven, not CoV-driven. +// Status values. See README.md "Acceptance gate" for why Status reads the +// paired delta CI (Distinguishable), not CoV. const ( - StatusOK = "ok" // Significant is true -- the delta is trustworthy regardless of routine CoV - StatusInsignificant = "insignificant" // Significant is false -- the delta does not clear its own measurement uncertainty; more Iterations or a quieter host shrink that uncertainty + StatusOK = "ok" // distinguishable AND clears the effect-size floor + StatusSubThreshold = "sub_threshold" // distinguishable but below the effect floor -- resolvable, but not meaningfully more expensive in ABSOLUTE time. NOT "correctly priced": a cheap op can still be mispriced. Only `ok` is correlation-eligible. + StatusInsignificant = "insignificant" // measurable but the CI straddles zero -- not distinguishable at this precision + StatusUnderpowered = "underpowered" // too few counts for a finite CI (count<2 or non-finite bound); the verdict short-circuits before distinguishability // StatusError is reserved for a per-case measurement failure. Not // currently produced: bench_test.go fails the whole benchmark loudly - // (b.Fatalf) on an invalid program rather than degrading to an error - // row, so every emitted Run today is OK or Insignificant. + // (b.Fatalf) on an invalid program rather than degrading to an error row. StatusError = "error" ) -// NewRun builds a Run from a differential result. c is the Case that -// produced d (for its Class); iterations is the sample count behind each -// series (baseline and target share one Config, so a single value applies -// to both). -func NewRun(c Case, d Diff, iterations int) Run { - cov := d.BaselineCoV - if d.TargetCoV > cov { - cov = d.TargetCoV - } - nvcsw := d.BaselineNvcsw - if d.TargetNvcsw > nvcsw { - nvcsw = d.TargetNvcsw - } - nivcsw := d.BaselineNivcsw - if d.TargetNivcsw > nivcsw { - nivcsw = d.TargetNivcsw +// distinguishable is the statistical half of the acceptance gate (F1): the +// paired delta CI excludes zero. It reads the RAW float64 bounds from crossRun +// (never the null-mapped *float64), so a non-finite bound is ±Inf-safe -- -Inf +// is not > 0 and +Inf is not < 0, so an underpowered row is never +// distinguishable and no nil deref is possible. It is deliberately NOT the +// advisory p_value. +func distinguishable(ciLo, ciHi float64) bool { return ciLo > 0 || ciHi < 0 } + +// effectSizePass is the practical half of the gate: the per-op median delta +// clears a minimum absolute-ns floor, so "significant" means "meaningfully more +// expensive," not merely "distinguishable at large n". A minPerOpNs <= 0 +// disables the floor (every distinguishable row passes). The gas-relative floor +// is deferred (see README/design). +func effectSizePass(perOpDeltaNs, minPerOpNs float64) bool { + return minPerOpNs <= 0 || math.Abs(perOpDeltaNs) >= minPerOpNs +} + +// classifyStatus derives the verdict. Underpowered short-circuits first (its CI +// is non-finite, so distinguishability is not meaningful). Acceptance +// (StatusOK) requires BOTH distinguishable and effectPass; distinguishable but +// below the floor is StatusSubThreshold. CoV/HighVariance never enter here. +func classifyStatus(underpowered, distinguishable, effectPass bool) string { + switch { + case underpowered: + return StatusUnderpowered + case distinguishable && effectPass: + return StatusOK + case distinguishable: + return StatusSubThreshold + default: + return StatusInsignificant } - status := StatusInsignificant - if d.Significant { - status = StatusOK +} + +// nullableBound maps a raw CI bound to the wire's nullable column: nil when +// non-finite (D-1: ±Inf must never reach encoding/json). This is the only +// place raw->nullable happens; the gate reads the raw bound upstream. +func nullableBound(x float64) *float64 { + if math.IsInf(x, 0) { + return nil } + return &x +} + +// hostHealth carries the advisory host-health signals for one opcode: the +// max-across-counts values from the collector. Named fields (like crossRunInput) +// keep the same-typed Nvcsw/Nivcsw from transposing at the NewRun call site. +type hostHealth struct { + CoV float64 // worse-of-pair within-run CoV, max across counts + HighVariance bool // CoV exceeded the ceiling on any count + Nvcsw int64 // worse-of-pair voluntary ctx switches, max across counts + Nivcsw int64 // worse-of-pair involuntary ctx switches, max across counts +} + +// NewRun composes the per-opcode aggregate row. cr carries the cross-run +// benchmath verdict (raw CI bounds); h carries the advisory host-health signals; +// minPerOpNs is the effect-size floor (<=0 disables it). The gate reads cr's raw +// bounds; the wire gets null-mapped ones. +func NewRun(c Case, cr crossRun, gasUsed uint64, count, iterations int, h hostHealth, minPerOpNs float64) Run { + // dist/effect (not the package funcs) to avoid shadowing. + dist := distinguishable(cr.CILo, cr.CIHi) + effect := effectSizePass(cr.MedianDeltaNs/float64(c.Reps), minPerOpNs) return Run{ - InputID: d.OpcodeID, - Class: string(c.Class), - Reps: d.Reps, - GasUsed: d.GasDelta, - ConstGas: c.ConstGas, - ExecTimeNs: d.DeltaNs, - Status: status, - Iterations: iterations, - CoV: cov, - Significant: d.Significant, - HighVariance: d.HighVariance, - Nvcsw: nvcsw, - Nivcsw: nivcsw, + InputID: c.OpcodeID, + Class: string(c.Class), + Reps: c.Reps, + GasUsed: gasUsed, + ConstGas: c.ConstGas, + ExecTimeNs: cr.MedianDeltaNs, + Count: count, + Iterations: iterations, + CILo: nullableBound(cr.CILo), + CIHi: nullableBound(cr.CIHi), + Confidence: cr.Confidence, + Distinguishable: dist, + EffectSizePass: effect, + PValue: cr.P, + Alpha: cr.Alpha, + Status: classifyStatus(cr.Underpowered, dist, effect), + CoV: h.CoV, + HighVariance: h.HighVariance, + Nvcsw: h.Nvcsw, + Nivcsw: h.Nivcsw, } } -var csvHeader = []string{"input_id", "class", "reps", "gas_used", "const_gas", "exec_time_ns", "status", "iterations", "cov", "significant", "high_variance", "nvcsw", "nivcsw"} +var csvHeader = []string{"input_id", "class", "reps", "gas_used", "const_gas", "exec_time_ns", "count", "iterations", "ci_lo", "ci_hi", "confidence", "distinguishable", "effect_size_pass", "p_value", "alpha", "status", "cov", "high_variance", "nvcsw", "nivcsw"} + +// formatBound renders a nullable CI bound: empty for a null (underpowered) +// bound, full precision otherwise. +func formatBound(p *float64) string { + if p == nil { + return "" + } + return strconv.FormatFloat(*p, 'g', -1, 64) +} // WriteCSV writes a header plus one row per run. func WriteCSV(w io.Writer, runs []Run) error { @@ -92,14 +165,23 @@ func WriteCSV(w io.Writer, runs []Run) error { strconv.FormatUint(r.GasUsed, 10), strconv.FormatUint(r.ConstGas, 10), strconv.FormatFloat(r.ExecTimeNs, 'g', -1, 64), - r.Status, + strconv.Itoa(r.Count), strconv.Itoa(r.Iterations), + // A null bound (underpowered) renders as an empty field; older CSV + // artifacts wrote +Inf here. See README.md schema. + formatBound(r.CILo), + formatBound(r.CIHi), + strconv.FormatFloat(r.Confidence, 'g', -1, 64), + strconv.FormatBool(r.Distinguishable), + strconv.FormatBool(r.EffectSizePass), + strconv.FormatFloat(r.PValue, 'g', -1, 64), + strconv.FormatFloat(r.Alpha, 'g', -1, 64), + r.Status, // CoV is advisory-only (README.md "Acceptance gate"), so this // deliberately rounds to 6 sig figs for readability; NDJSON's // json.Encoder gives CoV full precision instead -- a consumer // comparing CoV across the two formats will see this difference. strconv.FormatFloat(r.CoV, 'g', 6, 64), - strconv.FormatBool(r.Significant), strconv.FormatBool(r.HighVariance), strconv.FormatInt(r.Nvcsw, 10), strconv.FormatInt(r.Nivcsw, 10), diff --git a/tools/gasbench/emit_test.go b/tools/gasbench/emit_test.go index 3125fbc113..630fe5f899 100644 --- a/tools/gasbench/emit_test.go +++ b/tools/gasbench/emit_test.go @@ -1,41 +1,162 @@ package gasbench -import "testing" - -// TestNewRunStatusIsSignificantOnly pins that Status is a pure function of -// Significant: neither HighVariance nor the raw CoV values behind it may -// flip it. See README.md "Acceptance gate". -func TestNewRunStatusIsSignificantOnly(t *testing.T) { - c := Case{OpcodeID: "ADD", Class: ClassArithmetic} +import ( + "bytes" + "math" + "testing" +) +// TestDistinguishableReadsRawBounds pins the gate (F1): it reads the raw CI +// bounds and is ±Inf-safe -- -Inf is not > 0 and +Inf is not < 0, so an +// underpowered (non-finite) CI is never distinguishable. +func TestDistinguishableReadsRawBounds(t *testing.T) { cases := []struct { - significant bool - highVariance bool - cov float64 // fed into BaselineCoV/TargetCoV directly - want string + name string + lo, hi float64 + want bool }{ - {significant: true, highVariance: false, cov: 0, want: StatusOK}, - {significant: true, highVariance: true, cov: 0, want: StatusOK}, - {significant: false, highVariance: false, cov: 0, want: StatusInsignificant}, - {significant: false, highVariance: true, cov: 0, want: StatusInsignificant}, - // A regression that gated Status on CoV directly (rather than on - // Significant/HighVariance) would flip these two, even though the - // cases above never exercise a nonzero CoV. - {significant: true, highVariance: true, cov: 0.9, want: StatusOK}, - {significant: false, highVariance: true, cov: 0.9, want: StatusInsignificant}, + {"wholly above zero", 1, 3, true}, + {"wholly below zero", -3, -1, true}, + {"straddles zero", -1, 2, false}, + {"lower bound touches zero", 0, 5, false}, + {"underpowered ±Inf", math.Inf(-1), math.Inf(1), false}, + } + for _, tc := range cases { + if got := distinguishable(tc.lo, tc.hi); got != tc.want { + t.Errorf("%s: distinguishable(%v,%v) = %v, want %v", tc.name, tc.lo, tc.hi, got, tc.want) + } } +} +// TestClassifyStatus pins the verdict matrix: underpowered short-circuits +// first; ok requires BOTH distinguishable and effectPass; distinguishable but +// below the floor is sub_threshold; otherwise insignificant. +func TestClassifyStatus(t *testing.T) { + cases := []struct { + name string + underpowered, distinguishable, effectPass bool + want string + }{ + {"underpowered wins over all", true, true, true, StatusUnderpowered}, + {"underpowered wins even if not distinguishable", true, false, false, StatusUnderpowered}, + {"distinguishable + effect => ok", false, true, true, StatusOK}, + {"distinguishable but below floor => sub_threshold", false, true, false, StatusSubThreshold}, + {"not distinguishable => insignificant", false, false, true, StatusInsignificant}, + {"not distinguishable, no effect => insignificant", false, false, false, StatusInsignificant}, + } for _, tc := range cases { - d := Diff{ - Significant: tc.significant, - HighVariance: tc.highVariance, - BaselineCoV: tc.cov, - TargetCoV: tc.cov, + if got := classifyStatus(tc.underpowered, tc.distinguishable, tc.effectPass); got != tc.want { + t.Errorf("%s: classifyStatus(up=%v,dist=%v,eff=%v) = %q, want %q", + tc.name, tc.underpowered, tc.distinguishable, tc.effectPass, got, tc.want) } - got := NewRun(c, d, 0).Status - if got != tc.want { - t.Errorf("Significant=%v HighVariance=%v CoV=%v: Status = %q, want %q", - tc.significant, tc.highVariance, tc.cov, got, tc.want) + } +} + +// TestEffectSizePass pins the practical-half gate: absolute per-op ns clears the +// floor; a non-positive floor disables it. +func TestEffectSizePass(t *testing.T) { + cases := []struct { + name string + perOpDeltaNs, minPerOpNs float64 + want bool + }{ + {"above floor", 5, 1, true}, + {"below floor", 0.5, 1, false}, + {"at floor", 1, 1, true}, + {"negative delta above floor by magnitude", -5, 1, true}, + {"floor disabled (zero)", 0.001, 0, true}, + {"floor disabled (negative)", 0.001, -1, true}, + } + for _, tc := range cases { + if got := effectSizePass(tc.perOpDeltaNs, tc.minPerOpNs); got != tc.want { + t.Errorf("%s: effectSizePass(%v,%v) = %v, want %v", tc.name, tc.perOpDeltaNs, tc.minPerOpNs, got, tc.want) } } } + +// TestNewRunUnderpoweredNullCIAndNDJSONSafe covers D-1: a non-finite CI maps to +// nil (JSON null / CSV empty) and does not break WriteNDJSON with an +// UnsupportedValueError. The status short-circuits to underpowered. +func TestNewRunUnderpoweredNullCIAndNDJSONSafe(t *testing.T) { + c := Case{OpcodeID: "ADD", Class: ClassArithmetic, Reps: 1000, ConstGas: 3} + up := crossRun{MedianDeltaNs: 3, CILo: math.Inf(-1), CIHi: math.Inf(1), P: 1, Underpowered: true} + + r := NewRun(c, up, 100, 1, 20000, hostHealth{}, 1.0) + if r.CILo != nil || r.CIHi != nil { + t.Errorf("underpowered CI must map to nil, got Lo=%v Hi=%v", r.CILo, r.CIHi) + } + if r.Distinguishable { + t.Error("underpowered row must not be distinguishable") + } + if r.Status != StatusUnderpowered { + t.Errorf("status = %q, want %q", r.Status, StatusUnderpowered) + } + + var nd bytes.Buffer + if err := WriteNDJSON(&nd, []Run{r}); err != nil { + t.Fatalf("WriteNDJSON of an underpowered row must not error (D-1): %v", err) + } + var csvBuf bytes.Buffer + if err := WriteCSV(&csvBuf, []Run{r}); err != nil { + t.Fatalf("WriteCSV of an underpowered row must not error: %v", err) + } + if !bytes.Contains(csvBuf.Bytes(), []byte(",,")) { + t.Errorf("underpowered CSV must render empty CI fields, got:\n%s", csvBuf.String()) + } +} + +// TestNewRunDistinguishableFiniteCI pins the ok path: a finite CI excluding +// zero AND a per-op delta above the floor yields distinguishable + effect_size_pass +// + non-nil bounds + status ok. Per-op = 30000/1000 = 30ns >> the 1.0ns floor. +func TestNewRunDistinguishableFiniteCI(t *testing.T) { + c := Case{OpcodeID: "DIV", Class: ClassArithmetic, Reps: 1000, ConstGas: 5} + fin := crossRun{MedianDeltaNs: 30000, CILo: 28000, CIHi: 32000, Confidence: 0.95, P: 4e-5, Alpha: 0.05} + + r := NewRun(c, fin, 5000, 10, 20000, hostHealth{CoV: 0.01}, 1.0) + if r.CILo == nil || r.CIHi == nil { + t.Fatal("finite CI must be non-nil") + } + if !r.Distinguishable { + t.Error("CI excluding zero must be distinguishable") + } + if !r.EffectSizePass { + t.Error("30ns/op must clear the 1.0ns floor") + } + if r.Status != StatusOK { + t.Errorf("status = %q, want %q", r.Status, StatusOK) + } +} + +// TestNewRunEffectFloorDemotes (AC9): a distinguishable row whose per-op delta +// is below the floor is demoted to sub_threshold -- distinguishable, but not +// meaningfully more expensive. Per-op = 500/1000 = 0.5ns < the 1.0ns floor. +func TestNewRunEffectFloorDemotes(t *testing.T) { + c := Case{OpcodeID: "ADD", Class: ClassArithmetic, Reps: 1000, ConstGas: 3} + cr := crossRun{MedianDeltaNs: 500, CILo: 400, CIHi: 600, Confidence: 0.95, P: 1e-4, Alpha: 0.05} + + r := NewRun(c, cr, 1000, 10, 20000, hostHealth{}, 1.0) + if !r.Distinguishable { + t.Error("CI excluding zero must be distinguishable") + } + if r.EffectSizePass { + t.Error("0.5ns/op must NOT clear the 1.0ns floor") + } + if r.Status != StatusSubThreshold { + t.Errorf("status = %q, want %q", r.Status, StatusSubThreshold) + } +} + +// TestNewRunEffectFloorPasses (AC10): the same shape scaled above the floor is ok. +// Per-op = 5000/1000 = 5ns > the 1.0ns floor. +func TestNewRunEffectFloorPasses(t *testing.T) { + c := Case{OpcodeID: "ADD", Class: ClassArithmetic, Reps: 1000, ConstGas: 3} + cr := crossRun{MedianDeltaNs: 5000, CILo: 4000, CIHi: 6000, Confidence: 0.95, P: 1e-4, Alpha: 0.05} + + r := NewRun(c, cr, 1000, 10, 20000, hostHealth{}, 1.0) + if !r.EffectSizePass { + t.Error("5ns/op must clear the 1.0ns floor") + } + if r.Status != StatusOK { + t.Errorf("status = %q, want %q", r.Status, StatusOK) + } +} diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh index babcf8a1a2..30482a1b53 100755 --- a/tools/gasbench/run.sh +++ b/tools/gasbench/run.sh @@ -21,13 +21,20 @@ CORE="${GASBENCH_CORE:-3}" warn() { printf 'WARN: %s\n' "$*" >&2; } +checked_freq=0 if [ -r /sys/devices/system/cpu/intel_pstate/no_turbo ]; then + checked_freq=1 [ "$(cat /sys/devices/system/cpu/intel_pstate/no_turbo)" = "1" ] || warn "turbo appears ENABLED" fi gov="/sys/devices/system/cpu/cpu${CORE}/cpufreq/scaling_governor" if [ -r "$gov" ]; then + checked_freq=1 [ "$(cat "$gov")" = "performance" ] || warn "cpu${CORE} governor != performance" fi +# These sysfs paths are x86/Linux-specific and absent on ARM (Graviton) and +# non-Linux. Absence means "could not verify", NOT "clean" -- say so, or the +# operator reads silence as a passing check. +[ "$checked_freq" = "1" ] || warn "cannot verify turbo/governor here (no intel_pstate or cpufreq) -- trust numbers only on a known fixed-frequency host" export GOMAXPROCS=1 # See README.md "Running it" for the full env-var list and defaults. diff --git a/tools/gasbench/stats.go b/tools/gasbench/stats.go index 218b5894db..3e0c1636d3 100644 --- a/tools/gasbench/stats.go +++ b/tools/gasbench/stats.go @@ -8,16 +8,14 @@ import ( // Stats summarizes a sample series. All time fields are nanoseconds. type Stats struct { - N int - Min float64 // least-perturbed estimator for CPU-bound work; noise only adds time - Max float64 - Mean float64 - Median float64 - P99 float64 - Stddev float64 // sample stddev (n-1) - CoV float64 // Stddev/Mean; advisory only -- see Diff.Significant, README.md "Acceptance gate" - SEMean float64 // standard error of the mean - SEMedian float64 // asymptotic standard error of the median + N int + Min float64 // least-perturbed estimator for CPU-bound work; noise only adds time + Max float64 + Mean float64 + Median float64 + P99 float64 + Stddev float64 // sample stddev (n-1) + CoV float64 // Stddev/Mean; advisory only -- see README.md "Acceptance gate" } // Summarize computes the full stat set over the samples. @@ -48,9 +46,6 @@ func Summarize(samples []time.Duration) Stats { ss += dx * dx } st.Stddev = math.Sqrt(ss / float64(n-1)) - st.SEMean = st.Stddev / math.Sqrt(float64(n)) - // SE of the median for a roughly-normal center: sqrt(pi/2)*SEMean. - st.SEMedian = 1.2533141373155 * st.SEMean } if st.Mean > 0 { st.CoV = st.Stddev / st.Mean From bc7205f5a105be1eb458222e6b45b0cdbad8fdb7 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 09:05:16 -0700 Subject: [PATCH 13/19] test+docs(tools/gasbench): peer-sweep follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TestWriteRoundTripMixed: a finite-CI ok row and an underpowered null-CI row through both writers, with the NDJSON decoded back through a standard json.Decoder (finite bound -> float, null -> nil) and the CSV parsed to assert the number/empty rendering -- closes the round-trip clause of the underpowered acceptance criterion and the finite-CI writer path. - TestAnalyzeCrossRunUnderpowered now also pins the ±Inf raw bounds and that benchmath warnings are surfaced at n=1. - AGENTS.md file table: add the crossrun_test/diff_test/programs_test rows. - README: drop a stale "control" class from Scope (Specs has four families); document the CSV-vs-NDJSON CoV precision divergence in the schema table. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/AGENTS.md | 5 ++- tools/gasbench/README.md | 5 +-- tools/gasbench/crossrun_test.go | 6 ++++ tools/gasbench/emit_test.go | 58 +++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/tools/gasbench/AGENTS.md b/tools/gasbench/AGENTS.md index a7a7d72c22..fb6dc1889b 100644 --- a/tools/gasbench/AGENTS.md +++ b/tools/gasbench/AGENTS.md @@ -60,7 +60,10 @@ README.md "Running it" / "Output schema". | `program.go` | `Program`: warmed tracer-free EVM environment, one bytecode input | | `programs.go` | `OpSpec`/`Specs`/`Case`/`BuildCaseWith`: the differential bytecode construction | | `emit.go` | `Run`, the verdict (`distinguishable`/`classifyStatus`), CSV/NDJSON output | -| `emit_test.go` | pins `Run.Status`/CI as pure functions of the `crossRun` verdict (`classifyStatus`, `distinguishable`) | +| `emit_test.go` | pins `Run.Status`/CI as pure functions of the `crossRun` verdict (`classifyStatus`, `distinguishable`), the effect floor, and D-1 null-CI writer safety | +| `crossrun_test.go` | pins `analyzeCrossRun`: drift-survival (paired beats unpaired), straddles-zero, underpowered | +| `diff_test.go` | pins `Subtract`'s per-pass contract (delta/gas/per-op) | +| `programs_test.go` | pins the `BuildCaseWith` arity-0 panic guard | | `bench_test.go` | `BenchmarkOpcodes`: wires the above into `go test -bench` | | `run.sh` | pinned-core runner + operator checklist for turbo/governor/isolation | | `README.md` | operator quickstart + full rationale: construction, acceptance gate, diagnostics | diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index c6bbece16c..15c4ef5195 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -392,7 +392,7 @@ One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON. Columns are in | `p_value` | **advisory only:** unpaired Mann-Whitney U p over baseline vs target medians; NOT the gate | | `alpha` | `CompareAlpha` behind the advisory `p_value` | | `status` | `ok \| sub_threshold \| insignificant \| underpowered \| error` — see below; never gated on CoV | -| `cov` | worse (max) of the baseline/target series CoV, max across counts — advisory only | +| `cov` | worse (max) of the baseline/target series CoV, max across counts — advisory only. CSV rounds to 6 sig figs for readability; NDJSON carries full precision — expect tail digits to differ across formats | | `high_variance` | advisory: CoV exceeded `CoVCeiling` on any count | | `nvcsw` / `nivcsw` | worse (max) of the baseline/target voluntary/involuntary context-switch counts, max across counts | @@ -425,7 +425,8 @@ checklist a new `Specs` entry needs. ## Scope Cleanly-benchmarkable-as-a-scalar opcodes only (arithmetic/bitwise/ -comparison/stack/control). Deferred, not yet implemented: parametric-curve +comparison/stack — the four `Class` families in `Specs`). Deferred, not yet +implemented: parametric-curve opcodes (`KECCAK256`/`EXP`/copy/memory/`LOG` by size), the state-touching context-dependent matrix (`SLOAD`/`SSTORE` by warm/cold, `CALL` family, `CREATE`), real-block replay (F1), and Sei's custom precompiles (0x1001+, diff --git a/tools/gasbench/crossrun_test.go b/tools/gasbench/crossrun_test.go index 6ce54d0eb8..0c8d9409c9 100644 --- a/tools/gasbench/crossrun_test.go +++ b/tools/gasbench/crossrun_test.go @@ -79,4 +79,10 @@ func TestAnalyzeCrossRunUnderpowered(t *testing.T) { if cr.MedianDeltaNs != 3 { t.Errorf("point median of a single delta got %v, want 3", cr.MedianDeltaNs) } + if !math.IsInf(cr.CILo, -1) || !math.IsInf(cr.CIHi, 1) { + t.Errorf("n=1 CI bounds got [%v, %v], want ±Inf", cr.CILo, cr.CIHi) + } + if len(cr.Warnings) == 0 { + t.Error("n=1 must surface at least one benchmath warning") + } } diff --git a/tools/gasbench/emit_test.go b/tools/gasbench/emit_test.go index 630fe5f899..058352a090 100644 --- a/tools/gasbench/emit_test.go +++ b/tools/gasbench/emit_test.go @@ -2,6 +2,8 @@ package gasbench import ( "bytes" + "encoding/csv" + "encoding/json" "math" "testing" ) @@ -160,3 +162,59 @@ func TestNewRunEffectFloorPasses(t *testing.T) { t.Errorf("status = %q, want %q", r.Status, StatusOK) } } + +// TestWriteRoundTripMixed pins the writers on a mixed batch: a finite-CI ok row +// and an underpowered null-CI row. NDJSON must round-trip through a standard +// decoder (finite bound -> the float, null -> nil), and CSV must render the +// finite bound as a number and the null bound as an empty field. +func TestWriteRoundTripMixed(t *testing.T) { + okRun := NewRun( + Case{OpcodeID: "DIV", Class: ClassArithmetic, Reps: 1000, ConstGas: 5}, + crossRun{MedianDeltaNs: 72534, CILo: 72501, CIHi: 72558.5, Confidence: 0.978, P: 4e-5, Alpha: 0.05}, + 3000, 10, 20000, hostHealth{CoV: 0.01}, 1.0) + upRun := NewRun( + Case{OpcodeID: "ADD", Class: ClassArithmetic, Reps: 1000, ConstGas: 3}, + crossRun{MedianDeltaNs: 3, CILo: math.Inf(-1), CIHi: math.Inf(1), P: 1, Underpowered: true}, + 1000, 1, 20000, hostHealth{}, 1.0) + runs := []Run{okRun, upRun} + + var nd bytes.Buffer + if err := WriteNDJSON(&nd, runs); err != nil { + t.Fatalf("WriteNDJSON: %v", err) + } + dec := json.NewDecoder(&nd) + var got [2]Run + for i := range got { + if err := dec.Decode(&got[i]); err != nil { + t.Fatalf("decode NDJSON row %d: %v", i, err) + } + } + if got[0].CILo == nil || *got[0].CILo != 72501 { + t.Errorf("ok row ci_lo round-trip got %v, want 72501", got[0].CILo) + } + if got[1].CILo != nil || got[1].CIHi != nil { + t.Errorf("underpowered row CI must decode to nil, got %v/%v", got[1].CILo, got[1].CIHi) + } + if got[0].Status != StatusOK || got[1].Status != StatusUnderpowered { + t.Errorf("statuses got %q/%q, want %q/%q", got[0].Status, got[1].Status, StatusOK, StatusUnderpowered) + } + + var cb bytes.Buffer + if err := WriteCSV(&cb, runs); err != nil { + t.Fatalf("WriteCSV: %v", err) + } + recs, err := csv.NewReader(&cb).ReadAll() + if err != nil { + t.Fatalf("parse CSV: %v", err) + } + // header + 2 rows; ci_lo/ci_hi are columns 8/9 (0-indexed). + if len(recs) != 3 { + t.Fatalf("CSV records got %d, want 3", len(recs)) + } + if recs[1][8] != "72501" { + t.Errorf("ok row ci_lo CSV got %q, want %q", recs[1][8], "72501") + } + if recs[2][8] != "" || recs[2][9] != "" { + t.Errorf("underpowered row CI CSV must be empty, got %q/%q", recs[2][8], recs[2][9]) + } +} From d7689c2a5894fcc897cd11a9907e7fd57d406dad Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 10:15:32 -0700 Subject: [PATCH 14/19] fix(tools/gasbench): address PR review findings (NaN bounds, lint gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - nullableBound treats NaN as non-finite alongside ±Inf (encoding/json rejects both), and Underpowered detection uses the same finite() predicate — closes the D-1 gap Cursor flagged. benchmath's median CI emits ±Inf sentinels, never NaN, so this was latent, but the wire invariant is "non-finite never reaches the encoder," now enforced and unit-pinned (distinguishable(NaN,NaN)=false, nullableBound(NaN)=nil). - Drop the redundant int64() conversions in rusageSnapshot: Rusage.Nvcsw/Nivcsw are int64 on every target platform (darwin/arm64, linux/amd64, linux/arm64) — clears unconvert. - Guard BuildCaseWith's conversions for gosec G115: reps > 0 and Arity <= 16 (DUP16 is the deepest lift) now panic at construction, with nolint on the guarded sites. golangci-lint run ./tools/gasbench/...: 0 issues. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/crossrun.go | 3 +-- tools/gasbench/emit.go | 10 +++++++--- tools/gasbench/emit_test.go | 14 ++++++++++++++ tools/gasbench/gasbench.go | 2 +- tools/gasbench/programs.go | 12 ++++++++++-- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/tools/gasbench/crossrun.go b/tools/gasbench/crossrun.go index 67900172e6..ca1954c16d 100644 --- a/tools/gasbench/crossrun.go +++ b/tools/gasbench/crossrun.go @@ -1,7 +1,6 @@ package gasbench import ( - "math" "slices" "golang.org/x/perf/benchmath" @@ -53,7 +52,7 @@ func analyzeCrossRun(in crossRunInput, alpha, confidence float64) crossRun { Confidence: sum.Confidence, P: cmp.P, Alpha: cmp.Alpha, - Underpowered: len(in.Deltas) < 2 || math.IsInf(sum.Lo, 0) || math.IsInf(sum.Hi, 0), + Underpowered: len(in.Deltas) < 2 || !finite(sum.Lo) || !finite(sum.Hi), Warnings: slices.Concat(sum.Warnings, cmp.Warnings), } } diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index 3b775f3dae..3e81aa17ef 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -89,15 +89,19 @@ func classifyStatus(underpowered, distinguishable, effectPass bool) string { } // nullableBound maps a raw CI bound to the wire's nullable column: nil when -// non-finite (D-1: ±Inf must never reach encoding/json). This is the only -// place raw->nullable happens; the gate reads the raw bound upstream. +// non-finite (D-1: neither ±Inf nor NaN may reach encoding/json — it rejects +// both). This is the only place raw->nullable happens; the gate reads the raw +// bound upstream. func nullableBound(x float64) *float64 { - if math.IsInf(x, 0) { + if !finite(x) { return nil } return &x } +// finite reports whether x is a real number (not ±Inf, not NaN). +func finite(x float64) bool { return !math.IsInf(x, 0) && !math.IsNaN(x) } + // hostHealth carries the advisory host-health signals for one opcode: the // max-across-counts values from the collector. Named fields (like crossRunInput) // keep the same-typed Nvcsw/Nivcsw from transposing at the NewRun call site. diff --git a/tools/gasbench/emit_test.go b/tools/gasbench/emit_test.go index 058352a090..6bd4bfa209 100644 --- a/tools/gasbench/emit_test.go +++ b/tools/gasbench/emit_test.go @@ -22,6 +22,7 @@ func TestDistinguishableReadsRawBounds(t *testing.T) { {"straddles zero", -1, 2, false}, {"lower bound touches zero", 0, 5, false}, {"underpowered ±Inf", math.Inf(-1), math.Inf(1), false}, + {"NaN bounds", math.NaN(), math.NaN(), false}, } for _, tc := range cases { if got := distinguishable(tc.lo, tc.hi); got != tc.want { @@ -30,6 +31,19 @@ func TestDistinguishableReadsRawBounds(t *testing.T) { } } +// TestNullableBoundNonFinite pins D-1's full contract: every non-finite bound +// (±Inf and NaN — encoding/json rejects both) maps to nil; finite passes through. +func TestNullableBoundNonFinite(t *testing.T) { + for _, x := range []float64{math.Inf(1), math.Inf(-1), math.NaN()} { + if nullableBound(x) != nil { + t.Errorf("nullableBound(%v) must be nil", x) + } + } + if p := nullableBound(42.5); p == nil || *p != 42.5 { + t.Errorf("nullableBound(42.5) got %v, want 42.5", p) + } +} + // TestClassifyStatus pins the verdict matrix: underpowered short-circuits // first; ok requires BOTH distinguishable and effectPass; distinguishable but // below the floor is sub_threshold; otherwise insignificant. diff --git a/tools/gasbench/gasbench.go b/tools/gasbench/gasbench.go index 97643878ad..4de7a4119c 100644 --- a/tools/gasbench/gasbench.go +++ b/tools/gasbench/gasbench.go @@ -70,7 +70,7 @@ func rusageSnapshot() (nvcsw, nivcsw int64, err error) { if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { return 0, 0, fmt.Errorf("gasbench: getrusage: %w", err) } - return int64(ru.Nvcsw), int64(ru.Nivcsw), nil + return ru.Nvcsw, ru.Nivcsw, nil } // Measure runs Warmup discarded iterations, then Iterations timed iterations diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index 1563d20064..2e15268d8b 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -138,6 +138,14 @@ func BuildCaseWith(s OpSpec, reps int, operands []*uint256.Int) Case { if len(operands) < 2 || len(operands) < s.Arity { panic(fmt.Sprintf("gasbench: %s needs >= max(2, arity=%d) operands, got %d", s.Name, s.Arity, len(operands))) } + // Guard the conversions below: reps feeds uint64(reps) in ExpectedGasDelta + // and Arity feeds the DUP opcode byte (DUP16 is the EVM ceiling). + if reps <= 0 { + panic(fmt.Sprintf("gasbench: %s needs reps > 0, got %d", s.Name, reps)) + } + if s.Arity > 16 { + panic(fmt.Sprintf("gasbench: %s has Arity %d > 16 (DUP16 is the deepest lift)", s.Name, s.Arity)) + } var perUnitDelta uint64 switch s.Op { @@ -189,7 +197,7 @@ func BuildCaseWith(s OpSpec, reps int, operands []*uint256.Int) Case { // (equal operands would let uint256 short-circuit DIV/MOD -- see // seedOperands). GasFastestStep is identical for every DUP, so this // leaves the (n-1)*GasQuickStep gas algebra unchanged from a DUP1 fill. - dupN := vm.DUP1 + vm.OpCode(s.Arity-1) + dupN := vm.DUP1 + vm.OpCode(s.Arity-1) //nolint:gosec // 1 <= Arity <= 16 guarded above for d := 0; d < s.Arity; d++ { base.Push(operands[d]) tgt.Push(operands[d]) @@ -218,6 +226,6 @@ func BuildCaseWith(s OpSpec, reps int, operands []*uint256.Int) Case { ConstGas: s.ConstGas, Baseline: base.Bytes(), Target: tgt.Bytes(), - ExpectedGasDelta: perUnitDelta * uint64(reps), + ExpectedGasDelta: perUnitDelta * uint64(reps), //nolint:gosec // reps > 0 guarded above } } From 3b222a47e7b3fd62ea795d6d50b1579a990e7042 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 10:27:19 -0700 Subject: [PATCH 15/19] docs(tools/gasbench): EC2 recipe for the pinned-host run The README said what the host must be but not how to get one. Add the exact SSM-based c6g.metal workflow behind the shipped numbers: launch (AL2023 arm64, SSM instance profile, 60GB root), bootstrap (Go + clone + unit tests), the full run, result fetch, and the terminate reminder with cost/runtime expectations. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/README.md | 44 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index 15c4ef5195..d9846efee7 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -53,6 +53,50 @@ least setup because there is no turbo/governor to disable. Kernel-level isolation (`isolcpus` etc.) is deliberately NOT required — see "Acceptance gate" below for why plain pinning is enough. +
+EC2 recipe (the exact workflow behind the shipped numbers) + +A `c6g.metal` (Graviton2: fixed frequency, no SMT, no co-tenants) is the +least-setup host that satisfies the checklist. Ephemeral workflow via SSM — +no SSH keys, no inbound ports: + +```bash +# 1. Launch. Needs: an AL2023 arm64 AMI, any egress-capable subnet/SG, and an +# instance profile carrying AmazonSSMManagedInstanceCore. +AMI=$(aws ssm get-parameter \ + --name /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 \ + --query 'Parameter.Value' --output text) +IID=$(aws ec2 run-instances --image-id "$AMI" --instance-type c6g.metal \ + --subnet-id --security-group-ids \ + --iam-instance-profile Name= \ + --block-device-mappings 'DeviceName=/dev/xvda,Ebs={VolumeSize=60,VolumeType=gp3}' \ + --query 'Instances[0].InstanceId' --output text) + +# 2. Wait for SSM Online (bare metal boots in ~10 min), then bootstrap + run. +# Building sei-chain's module graph needs the 60GB root and a few minutes. +aws ssm send-command --instance-ids "$IID" --document-name AWS-RunShellScript \ + --parameters '{"commands":[ + "dnf install -y -q git tar gzip", + "curl -sL https://go.dev/dl/go1.25.6.linux-arm64.tar.gz | tar -C /usr/local -xz", + "export HOME=/root PATH=/usr/local/go/bin:$PATH", + "git clone --depth 1 https://github.com/sei-protocol/sei-chain.git /opt/sei-chain", + "cd /opt/sei-chain && go test -count=1 ./tools/gasbench/...", + "mkdir -p /opt/out && cd /opt/sei-chain && GASBENCH_OUT_CSV=/opt/out/gasbench.csv GASBENCH_OUT_NDJSON=/opt/out/gasbench.ndjson tools/gasbench/run.sh > /opt/out/run.log 2>&1" + ],"executionTimeout":["3600"]}' + +# 3. Fetch results (SSM output is text-only; base64 the CSV through it), +# then TERMINATE — the box bills ~$2.2/hr and the run needs ~15 min total. +# (send "base64 /opt/out/gasbench.csv", decode locally) +aws ec2 terminate-instances --instance-ids "$IID" +``` + +The full suite at defaults is ~11 min on this host; expect every row +`status=ok` with CI half-widths well under 1% of `exec_time_ns`. `run.sh` +prints `cannot verify turbo/governor` on Graviton — expected: there is no +turbo to check, which is the point of the host choice. + +
+ **3. Read the output:** each row is one opcode's *differential* cost — the target-minus-baseline delta, so setup/loop overhead has already cancelled (see "Differential construction"): From 0d9ea069accd7e28696cebfe0532dc6c6953e82a Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 10:35:30 -0700 Subject: [PATCH 16/19] fix(tools/gasbench): nolint the third guarded G115 site CI's gosec flags CI's gosec version also flags uint64(s.Arity-1) in perUnitDelta; same guard applies (Arity >= 1 panics above in the same branch), same annotation as the two sibling sites. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/programs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gasbench/programs.go b/tools/gasbench/programs.go index 2e15268d8b..8562f722c5 100644 --- a/tools/gasbench/programs.go +++ b/tools/gasbench/programs.go @@ -212,7 +212,7 @@ func BuildCaseWith(s OpSpec, reps int, operands []*uint256.Int) Case { base.Op(vm.POP) } } - perUnitDelta = s.ConstGas - uint64(s.Arity-1)*vm.GasQuickStep + perUnitDelta = s.ConstGas - uint64(s.Arity-1)*vm.GasQuickStep //nolint:gosec // 1 <= Arity <= 16 guarded above } base.Op(vm.STOP) From 970674949d8947d3a8073614b64672cc3c0040b2 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 10:39:42 -0700 Subject: [PATCH 17/19] fix(tools/gasbench): failed subtest emits a status=error row, not NaN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A b.Fatalf inside a b.Run subtest unwinds only that goroutine; the parent still aggregates the case's empty accumulator, and benchmath's median of an empty sample is NaN -- which corrupted the CSV ('NaN' string) and crashed WriteNDJSON at the end of the run, masking the original failure (verified empirically: empty accum -> Center=NaN, CI ±Inf). aggregate() now short- circuits an empty accumulator into NewErrorRun: all numerics finite/zero, CI null, status=error -- the row StatusError was reserved for. Unit-pinned incl. writer safety; README documents the status. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/README.md | 3 +++ tools/gasbench/bench_test.go | 8 ++++++++ tools/gasbench/emit.go | 23 ++++++++++++++++++++--- tools/gasbench/emit_test.go | 24 ++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index d9846efee7..d697cc42ca 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -145,6 +145,9 @@ target-minus-baseline delta, so setup/loop overhead has already cancelled default confidence — precisely: `<2` counts, or any count whose CI bound is non-finite at the requested confidence; see "Output schema"); `ci_lo`/`ci_hi` are null. Raise `GASBENCH_COUNT`. +- `status=error`: the case never produced a measurement — its subtest failed + (invalid program, `Measure` error) before accumulating a count. Numerics are + zero, CI null; see the test log for the underlying failure. - `high_variance=true`: advisory — the run saw unusual dispersion (noisy neighbor, throttling). It does not invalidate an `ok` result, and `nivcsw` tells you where to look: nonzero means the scheduler preempted diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index 22bce1ad9d..332c6681e8 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -129,6 +129,14 @@ type opcodeAccum struct { // analysis, and composes the per-opcode Run. Thin glue: the verdict lives in // NewRun/classifyStatus (emit.go). func aggregate(b *testing.B, acc *opcodeAccum, c Case, alpha, confidence, minPerOpNs float64) Run { + // A subtest that b.Fatalf'd (invalid program, Measure failure) unwinds only + // its own goroutine; the parent still aggregates. An empty accumulator + // through benchmath yields a NaN center that would corrupt the writers -- + // emit the error row instead. + if len(acc.deltas) == 0 { + b.Logf("%s: no measurements accumulated (subtest failed); emitting status=error row", c.OpcodeID) + return NewErrorRun(c, acc.iterations) + } cr := analyzeCrossRun(crossRunInput{ BaselineMedians: acc.baseMedians, TargetMedians: acc.tgtMedians, diff --git a/tools/gasbench/emit.go b/tools/gasbench/emit.go index 3e81aa17ef..6dcf9af37a 100644 --- a/tools/gasbench/emit.go +++ b/tools/gasbench/emit.go @@ -48,9 +48,10 @@ const ( StatusSubThreshold = "sub_threshold" // distinguishable but below the effect floor -- resolvable, but not meaningfully more expensive in ABSOLUTE time. NOT "correctly priced": a cheap op can still be mispriced. Only `ok` is correlation-eligible. StatusInsignificant = "insignificant" // measurable but the CI straddles zero -- not distinguishable at this precision StatusUnderpowered = "underpowered" // too few counts for a finite CI (count<2 or non-finite bound); the verdict short-circuits before distinguishability - // StatusError is reserved for a per-case measurement failure. Not - // currently produced: bench_test.go fails the whole benchmark loudly - // (b.Fatalf) on an invalid program rather than degrading to an error row. + // StatusError marks a case whose measurement never happened: its subtest + // b.Fatalf'd (invalid program, Measure failure), which unwinds only that + // subtest -- the parent still aggregates, and an empty accumulator must + // become an error row (NewErrorRun), never NaN in the output. StatusError = "error" ) @@ -102,6 +103,22 @@ func nullableBound(x float64) *float64 { // finite reports whether x is a real number (not ±Inf, not NaN). func finite(x float64) bool { return !math.IsInf(x, 0) && !math.IsNaN(x) } +// NewErrorRun is the row for a case whose measurement never happened (its +// subtest failed before accumulating a single count). Every numeric field is +// finite/zero and the CI is null, so the writers stay NaN-free (D-1) and a +// consumer filtering on status sees the failure instead of a corrupt number. +func NewErrorRun(c Case, iterations int) Run { + return Run{ + InputID: c.OpcodeID, + Class: string(c.Class), + Reps: c.Reps, + ConstGas: c.ConstGas, + Iterations: iterations, + PValue: 1, + Status: StatusError, + } +} + // hostHealth carries the advisory host-health signals for one opcode: the // max-across-counts values from the collector. Named fields (like crossRunInput) // keep the same-typed Nvcsw/Nivcsw from transposing at the NewRun call site. diff --git a/tools/gasbench/emit_test.go b/tools/gasbench/emit_test.go index 6bd4bfa209..5c23a2359c 100644 --- a/tools/gasbench/emit_test.go +++ b/tools/gasbench/emit_test.go @@ -31,6 +31,30 @@ func TestDistinguishableReadsRawBounds(t *testing.T) { } } +// TestNewErrorRunWriterSafe pins the failed-subtest path: an empty accumulator +// becomes an error row whose numerics are all finite/zero and whose CI is null, +// so a partially-failed run still emits parseable CSV/NDJSON (no NaN). +func TestNewErrorRunWriterSafe(t *testing.T) { + r := NewErrorRun(Case{OpcodeID: "ADD", Class: ClassArithmetic, Reps: 1000, ConstGas: 3}, 20000) + if r.Status != StatusError { + t.Errorf("status got %q, want %q", r.Status, StatusError) + } + if r.CILo != nil || r.CIHi != nil { + t.Error("error row CI must be null") + } + if !finite(r.ExecTimeNs) || r.Distinguishable || r.EffectSizePass { + t.Errorf("error row must be finite and non-passing: exec=%v dist=%v eff=%v", + r.ExecTimeNs, r.Distinguishable, r.EffectSizePass) + } + var nd, cb bytes.Buffer + if err := WriteNDJSON(&nd, []Run{r}); err != nil { + t.Fatalf("WriteNDJSON of an error row must not error: %v", err) + } + if err := WriteCSV(&cb, []Run{r}); err != nil { + t.Fatalf("WriteCSV of an error row must not error: %v", err) + } +} + // TestNullableBoundNonFinite pins D-1's full contract: every non-finite bound // (±Inf and NaN — encoding/json rejects both) maps to nil; finite passes through. func TestNullableBoundNonFinite(t *testing.T) { From 8ada691b2ae9e479499db0b27bcb510401aecd5c Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 12:19:06 -0700 Subject: [PATCH 18/19] fix(tools/gasbench): taint self-check failures; run.sh core-range fallback; README enum - A gas self-check b.Errorf (per-count algebra check or cross-count drift guard) does not unwind the subtest, so the invalid measurement was appended and aggregated into a normal-looking row. Both sites now taint the case (opcodeAccum.selfCheckFailed) and aggregate() routes a tainted case to the status=error row -- a construction that fails its own invariant can no longer masquerade as a trustworthy measurement. The b.Errorf stays, so the run still fails loudly. - run.sh validates GASBENCH_CORE against the host CPU count before pinning: taskset -c aborts outright on a nonexistent index (default CORE=3 on a 2-core runner killed the run before go test started), so out-of-range or non-numeric values now fall back to unpinned-with-warning, mirroring the no-taskset branch. The pinned-host path is unchanged. - README: the Output-schema `error` bullet said "not currently emitted", contradicting the code and the Quickstart since NewErrorRun landed; both bullets now describe the emitted behavior including the self-check-taint cause. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/README.md | 13 ++++++----- tools/gasbench/bench_test.go | 43 ++++++++++++++++++++++-------------- tools/gasbench/run.sh | 10 ++++++++- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index d697cc42ca..806fcd178e 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -145,9 +145,10 @@ target-minus-baseline delta, so setup/loop overhead has already cancelled default confidence — precisely: `<2` counts, or any count whose CI bound is non-finite at the requested confidence; see "Output schema"); `ci_lo`/`ci_hi` are null. Raise `GASBENCH_COUNT`. -- `status=error`: the case never produced a measurement — its subtest failed - (invalid program, `Measure` error) before accumulating a count. Numerics are - zero, CI null; see the test log for the underlying failure. +- `status=error`: the case never produced a trustworthy measurement — its + subtest failed before accumulating a count (invalid program, `Measure` + error), or a gas self-check fired. Numerics are zero, CI null; see the test + log for the underlying failure. - `high_variance=true`: advisory — the run saw unusual dispersion (noisy neighbor, throttling). It does not invalidate an `ok` result, and `nivcsw` tells you where to look: nonzero means the scheduler preempted @@ -454,8 +455,10 @@ One `Run` (`emit.go`) per opcode, written as CSV and/or NDJSON. Columns are in precision. - `underpowered` — too few counts for a finite CI (`count<2`, or a non-finite bound at the requested confidence); `ci_lo`/`ci_hi` are null. -- `error` — reserved for a per-case measurement failure; not currently emitted - (an invalid program fails the benchmark loudly instead). +- `error` — the case never produced a trustworthy measurement: its subtest + failed before accumulating a count (invalid program, `Measure` error), or a + gas self-check fired (the construction violated its own invariant). Numerics + are zero/finite, `ci_lo`/`ci_hi` null; the test log carries the cause. **Consumer note:** filter on `status` (only `ok` is correlation-eligible) and sign-check `exec_time_ns` (a negative value = a measured speedup). `ci_lo`/`ci_hi` diff --git a/tools/gasbench/bench_test.go b/tools/gasbench/bench_test.go index 332c6681e8..eceb2e3521 100644 --- a/tools/gasbench/bench_test.go +++ b/tools/gasbench/bench_test.go @@ -56,20 +56,24 @@ func BenchmarkOpcodes(b *testing.B) { // schema"), not of opcode timing. Insensitive to a // mis-transcribed OpSpec.Arity that still yields a // self-consistent ExpectedGasDelta -- see AGENTS.md. - if d.GasDelta != c.ExpectedGasDelta { - b.Errorf("%s: gas self-check failed: measured delta %d != expected %d (baseline/target pair does not isolate the opcode)", - c.OpcodeID, d.GasDelta, c.ExpectedGasDelta) - } - // Lock-free like sink (gasbench.go): subtests and -count reruns are // sequential (no b.Parallel). A future parallelize needs a mutex. acc := &collectors[i] acc.iterations = cfg.Iterations + if d.GasDelta != c.ExpectedGasDelta { + b.Errorf("%s: gas self-check failed: measured delta %d != expected %d (baseline/target pair does not isolate the opcode)", + c.OpcodeID, d.GasDelta, c.ExpectedGasDelta) + // b.Errorf does not unwind; taint the case so aggregate emits + // status=error instead of a normal-looking row built on a + // construction that failed its own invariant. + acc.selfCheckFailed = true + } // The gas delta is definitional and must be identical every count; // a drift means the differential construction is not deterministic. if acc.gasDeltaSet && d.GasDelta != acc.gasDelta { b.Errorf("%s: gas delta not identical across counts: %d != first-count %d", c.OpcodeID, d.GasDelta, acc.gasDelta) + acc.selfCheckFailed = true } if !acc.gasDeltaSet { acc.gasDelta, acc.gasDeltaSet = d.GasDelta, true @@ -113,16 +117,17 @@ func BenchmarkOpcodes(b *testing.B) { // crossRunInput instead. The Case is not stored here -- aggregation reads it // from cases[i]. type opcodeAccum struct { - iterations int - baseMedians []float64 // within-run median per count, ns - tgtMedians []float64 // within-run median per count, ns - deltas []float64 // per-count delta, ns (tgtMed - baseMed) - gasDelta uint64 // exact; guarded identical every count - gasDeltaSet bool - covMax float64 // running max of per-count worse-of-pair CoV (advisory) - highVariance bool // OR across counts (advisory) - nvcsw int64 // running max of worse-of-pair voluntary ctx switches - nivcsw int64 // running max of worse-of-pair involuntary ctx switches + iterations int + baseMedians []float64 // within-run median per count, ns + tgtMedians []float64 // within-run median per count, ns + deltas []float64 // per-count delta, ns (tgtMed - baseMed) + gasDelta uint64 // exact; guarded identical every count + gasDeltaSet bool + selfCheckFailed bool // a gas self-check b.Errorf fired; the case is tainted + covMax float64 // running max of per-count worse-of-pair CoV (advisory) + highVariance bool // OR across counts (advisory) + nvcsw int64 // running max of worse-of-pair voluntary ctx switches + nivcsw int64 // running max of worse-of-pair involuntary ctx switches } // aggregate builds the cross-run input from the collector, runs the benchmath @@ -133,8 +138,12 @@ func aggregate(b *testing.B, acc *opcodeAccum, c Case, alpha, confidence, minPer // its own goroutine; the parent still aggregates. An empty accumulator // through benchmath yields a NaN center that would corrupt the writers -- // emit the error row instead. - if len(acc.deltas) == 0 { - b.Logf("%s: no measurements accumulated (subtest failed); emitting status=error row", c.OpcodeID) + if len(acc.deltas) == 0 || acc.selfCheckFailed { + reason := "no measurements accumulated (subtest failed)" + if acc.selfCheckFailed { + reason = "gas self-check failed (construction invalid)" + } + b.Logf("%s: %s; emitting status=error row", c.OpcodeID, reason) return NewErrorRun(c, acc.iterations) } cr := analyzeCrossRun(crossRunInput{ diff --git a/tools/gasbench/run.sh b/tools/gasbench/run.sh index 30482a1b53..3b5a64dfe3 100755 --- a/tools/gasbench/run.sh +++ b/tools/gasbench/run.sh @@ -43,7 +43,15 @@ export GASBENCH_OUT_NDJSON="${GASBENCH_OUT_NDJSON:-gasbench.ndjson}" PIN=(env) if command -v taskset >/dev/null 2>&1; then - PIN=(taskset -c "$CORE") + # taskset -c N aborts outright if CPU index N does not exist (e.g. the + # default CORE=3 on a 2-core CI runner), so validate against the host and + # fall back to unpinned like the no-taskset branch instead of failing the run. + ncpu=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1) + if [ "$CORE" -ge 0 ] 2>/dev/null && [ "$CORE" -lt "$ncpu" ]; then + PIN=(taskset -c "$CORE") + else + warn "GASBENCH_CORE=$CORE out of range (host has $ncpu CPUs); running unpinned -- results are indicative only" + fi else warn "taskset not found (non-Linux?); running unpinned -- results are indicative only" fi From eb5795f3b6fb0cdc60682da6ec510f3bd56b2044 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Wed, 15 Jul 2026 12:41:34 -0700 Subject: [PATCH 19/19] docs(tools/gasbench): -benchtime=1x does not neutralize -count Third reviewer to conflate benchtime's within-pass iteration cap with the -count loop that sits outside it in processBench; document the two layers and the observable proof (count=1 -> underpowered rows; count=10 -> count=10 rows with finite CIs; -v shows K result lines per opcode). Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/gasbench/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/gasbench/README.md b/tools/gasbench/README.md index 806fcd178e..41269ee6f3 100644 --- a/tools/gasbench/README.md +++ b/tools/gasbench/README.md @@ -391,7 +391,14 @@ go test ./tools/gasbench/ -run '^$' -bench '^BenchmarkOpcodes$' \ `Measure`'s, not `b.N`'s, so `b.N` must stay at 1. `-count=K` reruns each opcode subtest K times **within the same OS process** (inside a single `BenchmarkOpcodes` invocation); benchmath folds those K passes into the one -aggregate row's cross-run CI. This is cross-run variance, not process-level +aggregate row's cross-run CI. `-benchtime=1x` does **not** neutralize +`-count`: the flags act at different layers — `benchtime` caps iterations +*within* one pass (`launch` adds none beyond `run1`'s single body execution), +while the `-count` loop sits *outside*, in `processBench`, re-running the +whole leaf benchmark K times — so each subtest body executes exactly K times +total. Observable either way: `-count=1` emits `count=1`/`underpowered` rows, +`-count=10` emits `count=10` rows with finite CIs, and `-v` shows K result +lines per opcode. This is cross-run variance, not process-level independence — true process-level independence would need K separate `go test` invocations. See the research doc for why this matters less than it sounds: JMH's fork/warmup/measurement-iteration model is the gold standard, but this