|
| 1 | +#!/usr/bin/env bash |
| 2 | +set -euo pipefail |
| 3 | + |
| 4 | +# Benchmark bgpkit-parser parsing performance using hyperfine. |
| 5 | +# Benchmarks the local target/release binary of bgpkit-parser and compares against the system PATH binary. |
| 6 | +# |
| 7 | +# Usage: |
| 8 | +# scripts/bench_hyperfine.sh [-- input-args] |
| 9 | +# Any arguments after -- are passed to the parser binaries before the INPUT file. |
| 10 | +# |
| 11 | +# Environment variables: |
| 12 | +# BGPKIT_BENCH_INPUT Path to input MRT/BGP file to parse. If unset, defaults to a large RIB. |
| 13 | +# HYPERFINE_RUNS Number of measurement runs (default: 3 for large RIB, else 10) |
| 14 | +# HYPERFINE_WARMUP Number of warmup runs (default: 1 for large RIB, else 3) |
| 15 | +# LOCAL_BIN Override local binary path (default: target/release/bgpkit-parser) |
| 16 | +# SYSTEM_BIN Override system binary path; default is the first bgpkit-parser found in PATH |
| 17 | +# HYPERFINE_EXTRA_ARGS Extra args passed to hyperfine (e.g., "--show-output") |
| 18 | +# BGPKIT_ARGS Extra args passed to bgpkit-parser before the INPUT (e.g., "--skip-v4") |
| 19 | +# OUT_DIR Output directory for results (default: benchmarks/hyperfine) |
| 20 | +# BENCH_DATA_DIR Directory to store/download inputs when BGPKIT_BENCH_INPUT is not set (default: benchmarks/test_data) |
| 21 | +# |
| 22 | +# Notes: |
| 23 | +# - This script assumes the bgpkit-parser CLI accepts a single positional INPUT path. |
| 24 | +# - If BGPKIT_BENCH_INPUT is not set, the script will: |
| 25 | +# 1) Use benchmarks/test_data/rib-example.bz2 (download if missing from https://spaces.bgpkit.org/parser/rib-example.bz2) |
| 26 | +# 2) Otherwise fall back to rib-example-small.bz2 in repo root or target/test_data. |
| 27 | +# Defaults to runs=3, warmup=1 when using the large RIB. |
| 28 | + |
| 29 | +ROOT_DIR="$(cd "$(dirname "$0")"/.. && pwd)" |
| 30 | +cd "$ROOT_DIR" |
| 31 | + |
| 32 | +# Check hyperfine availability |
| 33 | +if ! command -v hyperfine >/dev/null 2>&1; then |
| 34 | + echo "ERROR: 'hyperfine' is not installed or not in PATH." >&2 |
| 35 | + echo "Install: https://github.com/sharkdp/hyperfine or e.g. 'brew install hyperfine'" >&2 |
| 36 | + exit 1 |
| 37 | +fi |
| 38 | + |
| 39 | +# Build local release binary to ensure it's up to date |
| 40 | +LOCAL_BIN_DEFAULT="target/release/bgpkit-parser" |
| 41 | +echo "Building release binary..." |
| 42 | +cargo build --release >/dev/null |
| 43 | + |
| 44 | +LOCAL_BIN="${LOCAL_BIN:-$LOCAL_BIN_DEFAULT}" |
| 45 | +if [ ! -x "$LOCAL_BIN" ]; then |
| 46 | + echo "ERROR: Local binary not found or not executable: $LOCAL_BIN" >&2 |
| 47 | + exit 1 |
| 48 | +fi |
| 49 | + |
| 50 | +# Resolve system binary (for comparison) |
| 51 | +SYSTEM_BIN="${SYSTEM_BIN:-}" |
| 52 | +if [ -z "$SYSTEM_BIN" ]; then |
| 53 | + if command -v bgpkit-parser >/dev/null 2>&1; then |
| 54 | + SYSTEM_BIN="$(command -v bgpkit-parser)" |
| 55 | + else |
| 56 | + SYSTEM_BIN="" |
| 57 | + fi |
| 58 | +fi |
| 59 | + |
| 60 | +# Input file detection |
| 61 | +INPUT="${BGPKIT_BENCH_INPUT:-}" |
| 62 | +BENCH_DATA_DIR="${BENCH_DATA_DIR:-benchmarks/test_data}" |
| 63 | +LARGE_URL="https://spaces.bgpkit.org/parser/rib-example.bz2" |
| 64 | +LARGE_PATH="$BENCH_DATA_DIR/rib-example.bz2" |
| 65 | + |
| 66 | +if [ -z "$INPUT" ]; then |
| 67 | + # Prefer large RIB in benchmarks/test_data; download if missing |
| 68 | + if [ -f "$LARGE_PATH" ]; then |
| 69 | + INPUT="$LARGE_PATH" |
| 70 | + else |
| 71 | + mkdir -p "$BENCH_DATA_DIR" |
| 72 | + echo "Attempting to download large RIB to $LARGE_PATH" |
| 73 | + if command -v curl >/dev/null 2>&1; then |
| 74 | + curl -fL "$LARGE_URL" -o "$LARGE_PATH" |
| 75 | + elif command -v wget >/dev/null 2>&1; then |
| 76 | + wget -O "$LARGE_PATH" "$LARGE_URL" |
| 77 | + else |
| 78 | + echo "WARN: Neither curl nor wget available; cannot auto-download large RIB." >&2 |
| 79 | + fi |
| 80 | + if [ -f "$LARGE_PATH" ]; then |
| 81 | + INPUT="$LARGE_PATH" |
| 82 | + elif [ -f "rib-example-small.bz2" ]; then |
| 83 | + INPUT="rib-example-small.bz2" |
| 84 | + elif [ -f "target/test_data/rib-example-small.bz2" ]; then |
| 85 | + INPUT="target/test_data/rib-example-small.bz2" |
| 86 | + else |
| 87 | + echo "ERROR: Could not locate an input file." >&2 |
| 88 | + echo "Set BGPKIT_BENCH_INPUT to a valid file path, or place 'rib-example-small.bz2' in repo root, or run 'cargo bench' once to download test data, or install curl/wget for auto-download of the large RIB." >&2 |
| 89 | + exit 1 |
| 90 | + fi |
| 91 | + fi |
| 92 | +fi |
| 93 | + |
| 94 | +if [ ! -f "$INPUT" ]; then |
| 95 | + echo "ERROR: Input file does not exist: $INPUT" >&2 |
| 96 | + exit 1 |
| 97 | +fi |
| 98 | + |
| 99 | +# Determine defaults for runs/warmup based on input size choice |
| 100 | +if [[ "$INPUT" == *"rib-example.bz2"* ]]; then |
| 101 | + RUNS_DEFAULT=3 |
| 102 | + WARMUP_DEFAULT=1 |
| 103 | +else |
| 104 | + RUNS_DEFAULT=10 |
| 105 | + WARMUP_DEFAULT=3 |
| 106 | +fi |
| 107 | + |
| 108 | +# Optional args passed to parser binaries before INPUT |
| 109 | +FORWARDED_ARGS=() |
| 110 | +if [ $# -gt 0 ]; then |
| 111 | + # Support separator -- to pass arguments to the parser |
| 112 | + while [ $# -gt 0 ]; do |
| 113 | + if [ "$1" = "--" ]; then |
| 114 | + shift |
| 115 | + break |
| 116 | + fi |
| 117 | + shift |
| 118 | + done |
| 119 | + if [ $# -gt 0 ]; then |
| 120 | + FORWARDED_ARGS=("$@") |
| 121 | + fi |
| 122 | +fi |
| 123 | + |
| 124 | +# Merge BGPKIT_ARGS from env |
| 125 | +if [ -n "${BGPKIT_ARGS:-}" ]; then |
| 126 | + # shellcheck disable=SC2206 |
| 127 | + FORWARDED_ARGS=( ${BGPKIT_ARGS} "${FORWARDED_ARGS[@]}" ) |
| 128 | +fi |
| 129 | + |
| 130 | +RUNS="${HYPERFINE_RUNS:-$RUNS_DEFAULT}" |
| 131 | +WARMUP="${HYPERFINE_WARMUP:-$WARMUP_DEFAULT}" |
| 132 | +EXTRA="${HYPERFINE_EXTRA_ARGS:-}" |
| 133 | + |
| 134 | +# Output location |
| 135 | +STAMP="$(date +%Y%m%d-%H%M%S)" |
| 136 | +SHA="$(git rev-parse --short HEAD 2>/dev/null || echo nosha)" |
| 137 | +OUT_DIR="${OUT_DIR:-benchmarks/hyperfine}" |
| 138 | +mkdir -p "$OUT_DIR" |
| 139 | +OUT_JSON="$OUT_DIR/${STAMP}-${SHA}.json" |
| 140 | +OUT_MD="$OUT_DIR/${STAMP}-${SHA}.md" |
| 141 | + |
| 142 | + |
| 143 | +echo "Benchmarking with input: $INPUT" |
| 144 | +echo "Local bin: $LOCAL_BIN" |
| 145 | +if [ -n "$SYSTEM_BIN" ]; then |
| 146 | + echo "System bin: $SYSTEM_BIN" |
| 147 | +else |
| 148 | + echo "System bin: (not found in PATH)" |
| 149 | +fi |
| 150 | +echo "Runs: $RUNS, Warmup: $WARMUP" |
| 151 | + |
| 152 | +# Compare paths; warn if identical |
| 153 | +IDENTICAL=0 |
| 154 | +if [ -n "$SYSTEM_BIN" ]; then |
| 155 | + if [ "$SYSTEM_BIN" = "$LOCAL_BIN" ]; then |
| 156 | + IDENTICAL=1 |
| 157 | + fi |
| 158 | +fi |
| 159 | +if [ "$IDENTICAL" -eq 1 ]; then |
| 160 | + echo "WARN: system bgpkit-parser resolves to the same path as local: $SYSTEM_BIN" >&2 |
| 161 | +fi |
| 162 | + |
| 163 | +# Compose commands for hyperfine |
| 164 | +# Use explicit labels via --command-name for readability |
| 165 | +if [ -n "$SYSTEM_BIN" ] && [ -x "$SYSTEM_BIN" ] && [ "$IDENTICAL" -eq 0 ]; then |
| 166 | + hyperfine \ |
| 167 | + --warmup "$WARMUP" \ |
| 168 | + --runs "$RUNS" \ |
| 169 | + --export-json "$OUT_JSON" \ |
| 170 | + ${EXTRA} \ |
| 171 | + --command-name "local(target/release)" \ |
| 172 | + "\"$LOCAL_BIN\" ${FORWARDED_ARGS[*]-} \"$INPUT\"" \ |
| 173 | + --command-name "system(PATH)" \ |
| 174 | + "\"$SYSTEM_BIN\" ${FORWARDED_ARGS[*]-} \"$INPUT\"" |
| 175 | +else |
| 176 | + hyperfine \ |
| 177 | + --warmup "$WARMUP" \ |
| 178 | + --runs "$RUNS" \ |
| 179 | + --export-json "$OUT_JSON" \ |
| 180 | + ${EXTRA} \ |
| 181 | + --command-name "local(target/release)" \ |
| 182 | + "\"$LOCAL_BIN\" ${FORWARDED_ARGS[*]-} \"$INPUT\"" |
| 183 | +fi |
| 184 | + |
| 185 | +# Also write a small markdown summary next to the JSON for quick viewing |
| 186 | +{ |
| 187 | + echo "# Hyperfine: bgpkit-parser" |
| 188 | + echo |
| 189 | + echo "- Timestamp: $STAMP" |
| 190 | + echo "- Commit: $SHA" |
| 191 | + echo "- Input: $INPUT" |
| 192 | + echo "- Local: $LOCAL_BIN" |
| 193 | + if [ -n "$SYSTEM_BIN" ]; then |
| 194 | + echo "- System: $SYSTEM_BIN" |
| 195 | + fi |
| 196 | + echo "- Runs: $RUNS" |
| 197 | + echo "- Warmup: $WARMUP" |
| 198 | + echo |
| 199 | + echo "Command to reproduce:" |
| 200 | + echo "\n\tHYPERFINE_RUNS=$RUNS HYPERFINE_WARMUP=$WARMUP scripts/bench_hyperfine.sh -- ${FORWARDED_ARGS[*]-} \"$INPUT\"\n" |
| 201 | +} > "$OUT_MD" |
| 202 | + |
| 203 | +echo "Saved results:" |
| 204 | +echo " JSON: $OUT_JSON" |
| 205 | +echo " Note: $OUT_MD" |
0 commit comments