Skip to content

Commit 6611d19

Browse files
digizephdigizeph
committed
feat: add hyperfine benchmarking script for bgpkit-parser
- Implements `scripts/bench_hyperfine.sh` for benchmarking bgpkit-parser performance. - Compares local and system binaries using configurable input, environment variables, and hyperfine options. - Includes automatic input file setup and result outputs in JSON/Markdown formats.
1 parent 898eeb2 commit 6611d19

File tree

2 files changed

+209
-1
lines changed

2 files changed

+209
-1
lines changed

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,7 @@ examples/debug-file.rs
1414
.env
1515
.claude
1616
*.pcap
17-
*.bin
17+
*.bin
18+
19+
# local benchmark results
20+
benchmarks

‎scripts/bench_hyperfine.sh‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Benchmark bgpkit-parser parsing performance using hyperfine.
5+
# Benchmarks the local target/release binary of bgpkit-parser and compares against the system PATH binary.
6+
#
7+
# Usage:
8+
# scripts/bench_hyperfine.sh [-- input-args]
9+
# Any arguments after -- are passed to the parser binaries before the INPUT file.
10+
#
11+
# Environment variables:
12+
# BGPKIT_BENCH_INPUT Path to input MRT/BGP file to parse. If unset, defaults to a large RIB.
13+
# HYPERFINE_RUNS Number of measurement runs (default: 3 for large RIB, else 10)
14+
# HYPERFINE_WARMUP Number of warmup runs (default: 1 for large RIB, else 3)
15+
# LOCAL_BIN Override local binary path (default: target/release/bgpkit-parser)
16+
# SYSTEM_BIN Override system binary path; default is the first bgpkit-parser found in PATH
17+
# HYPERFINE_EXTRA_ARGS Extra args passed to hyperfine (e.g., "--show-output")
18+
# BGPKIT_ARGS Extra args passed to bgpkit-parser before the INPUT (e.g., "--skip-v4")
19+
# OUT_DIR Output directory for results (default: benchmarks/hyperfine)
20+
# BENCH_DATA_DIR Directory to store/download inputs when BGPKIT_BENCH_INPUT is not set (default: benchmarks/test_data)
21+
#
22+
# Notes:
23+
# - This script assumes the bgpkit-parser CLI accepts a single positional INPUT path.
24+
# - If BGPKIT_BENCH_INPUT is not set, the script will:
25+
# 1) Use benchmarks/test_data/rib-example.bz2 (download if missing from https://spaces.bgpkit.org/parser/rib-example.bz2)
26+
# 2) Otherwise fall back to rib-example-small.bz2 in repo root or target/test_data.
27+
# Defaults to runs=3, warmup=1 when using the large RIB.
28+
29+
ROOT_DIR="$(cd "$(dirname "$0")"/.. && pwd)"
30+
cd "$ROOT_DIR"
31+
32+
# Check hyperfine availability
33+
if ! command -v hyperfine >/dev/null 2>&1; then
34+
echo "ERROR: 'hyperfine' is not installed or not in PATH." >&2
35+
echo "Install: https://github.com/sharkdp/hyperfine or e.g. 'brew install hyperfine'" >&2
36+
exit 1
37+
fi
38+
39+
# Build local release binary to ensure it's up to date
40+
LOCAL_BIN_DEFAULT="target/release/bgpkit-parser"
41+
echo "Building release binary..."
42+
cargo build --release >/dev/null
43+
44+
LOCAL_BIN="${LOCAL_BIN:-$LOCAL_BIN_DEFAULT}"
45+
if [ ! -x "$LOCAL_BIN" ]; then
46+
echo "ERROR: Local binary not found or not executable: $LOCAL_BIN" >&2
47+
exit 1
48+
fi
49+
50+
# Resolve system binary (for comparison)
51+
SYSTEM_BIN="${SYSTEM_BIN:-}"
52+
if [ -z "$SYSTEM_BIN" ]; then
53+
if command -v bgpkit-parser >/dev/null 2>&1; then
54+
SYSTEM_BIN="$(command -v bgpkit-parser)"
55+
else
56+
SYSTEM_BIN=""
57+
fi
58+
fi
59+
60+
# Input file detection
61+
INPUT="${BGPKIT_BENCH_INPUT:-}"
62+
BENCH_DATA_DIR="${BENCH_DATA_DIR:-benchmarks/test_data}"
63+
LARGE_URL="https://spaces.bgpkit.org/parser/rib-example.bz2"
64+
LARGE_PATH="$BENCH_DATA_DIR/rib-example.bz2"
65+
66+
if [ -z "$INPUT" ]; then
67+
# Prefer large RIB in benchmarks/test_data; download if missing
68+
if [ -f "$LARGE_PATH" ]; then
69+
INPUT="$LARGE_PATH"
70+
else
71+
mkdir -p "$BENCH_DATA_DIR"
72+
echo "Attempting to download large RIB to $LARGE_PATH"
73+
if command -v curl >/dev/null 2>&1; then
74+
curl -fL "$LARGE_URL" -o "$LARGE_PATH"
75+
elif command -v wget >/dev/null 2>&1; then
76+
wget -O "$LARGE_PATH" "$LARGE_URL"
77+
else
78+
echo "WARN: Neither curl nor wget available; cannot auto-download large RIB." >&2
79+
fi
80+
if [ -f "$LARGE_PATH" ]; then
81+
INPUT="$LARGE_PATH"
82+
elif [ -f "rib-example-small.bz2" ]; then
83+
INPUT="rib-example-small.bz2"
84+
elif [ -f "target/test_data/rib-example-small.bz2" ]; then
85+
INPUT="target/test_data/rib-example-small.bz2"
86+
else
87+
echo "ERROR: Could not locate an input file." >&2
88+
echo "Set BGPKIT_BENCH_INPUT to a valid file path, or place 'rib-example-small.bz2' in repo root, or run 'cargo bench' once to download test data, or install curl/wget for auto-download of the large RIB." >&2
89+
exit 1
90+
fi
91+
fi
92+
fi
93+
94+
if [ ! -f "$INPUT" ]; then
95+
echo "ERROR: Input file does not exist: $INPUT" >&2
96+
exit 1
97+
fi
98+
99+
# Determine defaults for runs/warmup based on input size choice
100+
if [[ "$INPUT" == *"rib-example.bz2"* ]]; then
101+
RUNS_DEFAULT=3
102+
WARMUP_DEFAULT=1
103+
else
104+
RUNS_DEFAULT=10
105+
WARMUP_DEFAULT=3
106+
fi
107+
108+
# Optional args passed to parser binaries before INPUT
109+
FORWARDED_ARGS=()
110+
if [ $# -gt 0 ]; then
111+
# Support separator -- to pass arguments to the parser
112+
while [ $# -gt 0 ]; do
113+
if [ "$1" = "--" ]; then
114+
shift
115+
break
116+
fi
117+
shift
118+
done
119+
if [ $# -gt 0 ]; then
120+
FORWARDED_ARGS=("$@")
121+
fi
122+
fi
123+
124+
# Merge BGPKIT_ARGS from env
125+
if [ -n "${BGPKIT_ARGS:-}" ]; then
126+
# shellcheck disable=SC2206
127+
FORWARDED_ARGS=( ${BGPKIT_ARGS} "${FORWARDED_ARGS[@]}" )
128+
fi
129+
130+
RUNS="${HYPERFINE_RUNS:-$RUNS_DEFAULT}"
131+
WARMUP="${HYPERFINE_WARMUP:-$WARMUP_DEFAULT}"
132+
EXTRA="${HYPERFINE_EXTRA_ARGS:-}"
133+
134+
# Output location
135+
STAMP="$(date +%Y%m%d-%H%M%S)"
136+
SHA="$(git rev-parse --short HEAD 2>/dev/null || echo nosha)"
137+
OUT_DIR="${OUT_DIR:-benchmarks/hyperfine}"
138+
mkdir -p "$OUT_DIR"
139+
OUT_JSON="$OUT_DIR/${STAMP}-${SHA}.json"
140+
OUT_MD="$OUT_DIR/${STAMP}-${SHA}.md"
141+
142+
143+
echo "Benchmarking with input: $INPUT"
144+
echo "Local bin: $LOCAL_BIN"
145+
if [ -n "$SYSTEM_BIN" ]; then
146+
echo "System bin: $SYSTEM_BIN"
147+
else
148+
echo "System bin: (not found in PATH)"
149+
fi
150+
echo "Runs: $RUNS, Warmup: $WARMUP"
151+
152+
# Compare paths; warn if identical
153+
IDENTICAL=0
154+
if [ -n "$SYSTEM_BIN" ]; then
155+
if [ "$SYSTEM_BIN" = "$LOCAL_BIN" ]; then
156+
IDENTICAL=1
157+
fi
158+
fi
159+
if [ "$IDENTICAL" -eq 1 ]; then
160+
echo "WARN: system bgpkit-parser resolves to the same path as local: $SYSTEM_BIN" >&2
161+
fi
162+
163+
# Compose commands for hyperfine
164+
# Use explicit labels via --command-name for readability
165+
if [ -n "$SYSTEM_BIN" ] && [ -x "$SYSTEM_BIN" ] && [ "$IDENTICAL" -eq 0 ]; then
166+
hyperfine \
167+
--warmup "$WARMUP" \
168+
--runs "$RUNS" \
169+
--export-json "$OUT_JSON" \
170+
${EXTRA} \
171+
--command-name "local(target/release)" \
172+
"\"$LOCAL_BIN\" ${FORWARDED_ARGS[*]-} \"$INPUT\"" \
173+
--command-name "system(PATH)" \
174+
"\"$SYSTEM_BIN\" ${FORWARDED_ARGS[*]-} \"$INPUT\""
175+
else
176+
hyperfine \
177+
--warmup "$WARMUP" \
178+
--runs "$RUNS" \
179+
--export-json "$OUT_JSON" \
180+
${EXTRA} \
181+
--command-name "local(target/release)" \
182+
"\"$LOCAL_BIN\" ${FORWARDED_ARGS[*]-} \"$INPUT\""
183+
fi
184+
185+
# Also write a small markdown summary next to the JSON for quick viewing
186+
{
187+
echo "# Hyperfine: bgpkit-parser"
188+
echo
189+
echo "- Timestamp: $STAMP"
190+
echo "- Commit: $SHA"
191+
echo "- Input: $INPUT"
192+
echo "- Local: $LOCAL_BIN"
193+
if [ -n "$SYSTEM_BIN" ]; then
194+
echo "- System: $SYSTEM_BIN"
195+
fi
196+
echo "- Runs: $RUNS"
197+
echo "- Warmup: $WARMUP"
198+
echo
199+
echo "Command to reproduce:"
200+
echo "\n\tHYPERFINE_RUNS=$RUNS HYPERFINE_WARMUP=$WARMUP scripts/bench_hyperfine.sh -- ${FORWARDED_ARGS[*]-} \"$INPUT\"\n"
201+
} > "$OUT_MD"
202+
203+
echo "Saved results:"
204+
echo " JSON: $OUT_JSON"
205+
echo " Note: $OUT_MD"

0 commit comments

Comments
 (0)