Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 170 additions & 0 deletions .github/scripts/format_benchstat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""
Formats raw Go benchstat text output into a clean, executive GitHub Step Summary Markdown report.
"""
import sys
import re

def parse_val_unit(s):
"""Parses value string like '59.30m', '7.601Mi', '18.84k' into a numeric float."""
s = s.strip()
m = re.match(r'^([0-9.]+)([a-zA-Z%]*)$', s)
if not m:
return 0.0
val_str, unit = m.groups()
val = float(val_str)
multiplier = 1.0
if unit == 'k': multiplier = 1e3
elif unit in ('M', 'Mi'): multiplier = 1e6
elif unit in ('G', 'Gi'): multiplier = 1e9
elif unit == 'm': multiplier = 1e-3
elif unit in ('u', 'µ'): multiplier = 1e-6
elif unit == 'n': multiplier = 1e-9
return val * multiplier

def compute_pct(old_s, new_s):
old_v = parse_val_unit(old_s)
new_v = parse_val_unit(new_s)
if old_v == 0:
return 0.0
return ((new_v - old_v) / old_v) * 100.0

def format_benchstat(text):
lines = text.splitlines()
current_metric = None
geomeans = {}
fixtures = {} # fix_name -> {'time': (old, new), 'mem': (old, new), 'allocs': (old, new)}

for line in lines:
line_str = line.strip()
if 'sec/op' in line_str and 'vs base' in line_str:
current_metric = 'time'
elif 'B/op' in line_str and 'vs base' in line_str:
current_metric = 'mem'
elif 'allocs/op' in line_str and 'vs base' in line_str:
current_metric = 'allocs'
elif line_str.startswith('geomean'):
parts = line_str.split()
if len(parts) >= 3 and current_metric:
delta = parts[3] if len(parts) >= 4 else "~"
geomeans[current_metric] = (parts[1], parts[2], delta)
elif line_str.startswith('Pipeline/'):
parts = line_str.split()
if len(parts) >= 3 and current_metric:
fixture_raw = parts[0]
fix_name = fixture_raw.replace('Pipeline/', '')
if '-' in fix_name:
fix_name = fix_name.rsplit('-', 1)[0]

vals = [p for p in parts[1:] if re.match(r'^[0-9.]+[a-zA-Z]*$', p)]
if len(vals) >= 2:
old_v, new_v = vals[0], vals[1]
if fix_name not in fixtures:
fixtures[fix_name] = {}
fixtures[fix_name][current_metric] = (old_v, new_v)

# Build Markdown Output
md = []
md.append("### 📊 Benchmark A/B Comparison Report")
md.append("")
md.append("#### 📈 Key Metrics Summary (Geometric Mean)")
md.append("| Metric | Baseline | Current | Delta | Status |")
md.append("| :--- | :---: | :---: | :---: | :---: |")

metric_meta = {
'time': ('Execution Time', 'sec/op'),
'mem': ('Memory Allocated', 'B/op'),
'allocs': ('Allocations', 'allocs/op')
}

for key in ['time', 'mem', 'allocs']:
if key in geomeans:
old_g, new_g, _ = geomeans[key]
pct = compute_pct(old_g, new_g)
name, _ = metric_meta[key]

if abs(pct) < 0.5:
status = "➖ Neutral"
badge = f"`{pct:+.2f}%`"
elif pct < 0:
status = "🟢 Faster" if key == 'time' else "🟢 Reduced"
badge = f"**`{pct:+.2f}%`**"
else:
status = "🔴 Slower" if key == 'time' else "🟡 Increased"
badge = f"**`{pct:+.2f}%`**"

md.append(f"| **{name}** | `{old_g}` | `{new_g}` | {badge} | {status} |")

md.append("")

# Sort changes by time delta
changes = []
for fix_name, data in fixtures.items():
if 'time' in data:
old_t, new_t = data['time']
pct = compute_pct(old_t, new_t)
changes.append((fix_name, pct, old_t, new_t, data.get('mem', ('-', '-'))))

changes.sort(key=lambda x: x[1])

speedups = [c for c in changes if c[1] <= -10.0]
regressions = [c for c in changes if c[1] >= 10.0]

if speedups or regressions:
md.append("#### ⚡ Notable Performance Shifts (≥ 10% change)")
md.append("| Fixture | Baseline Time | Current Time | Time Delta | Memory Delta |")
md.append("| :--- | :---: | :---: | :---: | :---: |")

for fix_name, pct, old_t, new_t, mem_tuple in speedups + regressions:
icon = "🚀" if pct < 0 else "⚠️"
old_m, new_m = mem_tuple
mem_pct_str = "-"
if old_m != '-' and new_m != '-':
m_pct = compute_pct(old_m, new_m)
mem_pct_str = f"`{m_pct:+.1f}%`"
md.append(f"| {icon} `{fix_name}` | `{old_t}` | `{new_t}` | **`{pct:+.1f}%`** | {mem_pct_str} |")
md.append("")

# Collapsible Table for All Fixtures
md.append("<details>")
md.append(f"<summary><b>🔍 Full Breakdown ({len(fixtures)} Fixtures)</b></summary>")
md.append("")
md.append("| Fixture | Baseline Time | Current Time | Time Delta | Baseline Mem | Current Mem | Mem Delta |")
md.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |")

for fix_name, data in sorted(fixtures.items()):
old_t, new_t = data.get('time', ('-', '-'))
old_m, new_m = data.get('mem', ('-', '-'))

t_pct_str = "`~`"
if old_t != '-' and new_t != '-':
t_pct = compute_pct(old_t, new_t)
t_pct_str = f"`{t_pct:+.1f}%`" if abs(t_pct) >= 0.5 else "`~`"

m_pct_str = "`~`"
if old_m != '-' and new_m != '-':
m_pct = compute_pct(old_m, new_m)
m_pct_str = f"`{m_pct:+.1f}%`" if abs(m_pct) >= 0.5 else "`~`"

md.append(f"| `{fix_name}` | `{old_t}` | `{new_t}` | {t_pct_str} | `{old_m}` | `{new_m}` | {m_pct_str} |")

md.append("")
md.append("</details>")
md.append("")
md.append("<details>")
md.append("<summary><b>📄 Raw benchstat Console Output</b></summary>")
md.append("")
md.append("```")
md.append(text.strip())
md.append("```")
md.append("</details>")

return "\n".join(md)

if __name__ == '__main__':
if len(sys.argv) > 1:
with open(sys.argv[1], 'r') as f:
content = f.read()
else:
content = sys.stdin.read()
print(format_benchstat(content))
156 changes: 119 additions & 37 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,9 @@ concurrency:
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
test:
name: Test & Build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
lint:
name: Lint & Format Check
runs-on: ubuntu-latest

steps:
- name: Checkout code
Expand All @@ -37,40 +33,88 @@ jobs:
cache-dependency-path: 'go.sum'

- name: Lint (golangci-lint)
if: matrix.os == 'ubuntu-latest'
uses: golangci/golangci-lint-action@v9
with:
version: v2.12
args: --timeout=5m

- name: Verify formatting (golangci-lint fmt)
if: matrix.os == 'ubuntu-latest'
run: golangci-lint fmt --diff ./...

test-linux:
name: Test & Coverage (Linux)
needs: lint
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
cache: true
cache-dependency-path: 'go.sum'

- name: Install gotestsum
run: go install gotest.tools/gotestsum@latest

- name: Run test suite
if: matrix.os != 'ubuntu-latest'
run: gotestsum --format pkgname --junitfile unit-tests.xml -- ./...

- name: Run test suite with coverage
if: matrix.os == 'ubuntu-latest'
run: >-
gotestsum --format pkgname --junitfile unit-tests.xml --
-coverprofile=coverage.out -covermode=atomic -coverpkg=./...
./...

- name: Print coverage summary
if: matrix.os == 'ubuntu-latest'
run: go tool cover -func=coverage.out | tail -1
- name: Print & publish coverage summary
run: |
go tool cover -func=coverage.out | tail -1
echo "### 🧪 Code Coverage" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
go tool cover -func=coverage.out | tail -15 >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
go tool cover -html=coverage.out -o coverage.html

- name: Upload coverage report
if: matrix.os == 'ubuntu-latest'
uses: actions/upload-artifact@v4
with:
name: coverage-report
path: coverage.out
path: |
coverage.out
coverage.html

- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
if: always()
with:
report_paths: '**/unit-tests.xml'
check_name: 'Test Report (ubuntu-latest)'

test-cross:
name: Cross-Platform Test
needs: lint
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest]

steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
cache: true
cache-dependency-path: 'go.sum'

- name: Install gotestsum
run: go install gotest.tools/gotestsum@latest

- name: Run test suite
run: gotestsum --format pkgname --junitfile unit-tests.xml -- ./...

- name: Publish Test Report
uses: mikepenz/action-junit-report@v4
Expand All @@ -79,26 +123,64 @@ jobs:
report_paths: '**/unit-tests.xml'
check_name: 'Test Report (${{ matrix.os }})'

- name: Benchmark smoke test
if: matrix.os == 'ubuntu-latest'
run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=1 -benchtime=1x
benchmark:
name: Benchmark A/B Comparison
needs: [lint, test-linux]
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Check performance-critical file changes
id: filter
run: |
BASE_REF="${{ github.event.pull_request.base.sha || 'HEAD~1' }}"
CHANGED=$(git diff --name-only $BASE_REF HEAD || true)
echo "Changed files:"
echo "$CHANGED"
if echo "$CHANGED" | grep -qE '^(internal/engine/|internal/treesitter/|internal/actions/|tests/integration/|go\.mod|go\.sum)'; then
echo "::notice::Performance-critical files changed. Running A/B benchmarks..."
echo "run_benchmarks=true" >> $GITHUB_OUTPUT
else
echo "::notice::No performance-critical files changed. Skipping benchmarks."
echo "run_benchmarks=false" >> $GITHUB_OUTPUT
fi

- name: Set up Go
if: steps.filter.outputs.run_benchmarks == 'true'
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
cache: true
cache-dependency-path: 'go.sum'

- name: Install benchstat
if: steps.filter.outputs.run_benchmarks == 'true'
run: go install golang.org/x/perf/cmd/benchstat@latest

- name: Run benchmarks on current HEAD
if: steps.filter.outputs.run_benchmarks == 'true'
run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_new.txt

- name: Benchmark A/B comparison (benchstat & GITHUB_STEP_SUMMARY)
if: matrix.os == 'ubuntu-latest'
- name: Fetch baseline branch
if: steps.filter.outputs.run_benchmarks == 'true'
run: |
go install golang.org/x/perf/cmd/benchstat@latest
echo "==> Running benchmarks on current HEAD..."
go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_new.txt
echo "==> Fetching baseline..."
git fetch origin ${{ github.event.pull_request.base.ref || 'main' }} --depth=2 || true
BASE_REF="${{ github.event.pull_request.base.sha || 'HEAD~1' }}"
git checkout $BASE_REF || true
echo "==> Running benchmarks on baseline ($BASE_REF)..."

- name: Run benchmarks on baseline
if: steps.filter.outputs.run_benchmarks == 'true'
run: |
go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true

# Render Markdown comparison table for GitHub Actions Summary Tab
echo "### 📊 Benchmark A/B Comparison vs Baseline" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
benchstat bench_old.txt bench_new.txt >> $GITHUB_STEP_SUMMARY || true
echo '```' >> $GITHUB_STEP_SUMMARY
rm -f bench_old.txt bench_new.txt
echo "==> Restoring PR HEAD commit to access scripts..."
git checkout ${{ github.sha }} || git checkout - || true

- name: Render Benchmark A/B Summary
if: steps.filter.outputs.run_benchmarks == 'true'
run: |
benchstat bench_old.txt bench_new.txt > benchstat_raw.txt || true
python3 .github/scripts/format_benchstat.py benchstat_raw.txt >> $GITHUB_STEP_SUMMARY || cat benchstat_raw.txt >> $GITHUB_STEP_SUMMARY
rm -f bench_old.txt bench_new.txt benchstat_raw.txt
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/mattn/go-isatty v0.0.20
github.com/odvcencio/gotreesitter v0.16.0
github.com/odvcencio/gotreesitter v0.48.0
github.com/spf13/cobra v1.10.2
gopkg.in/yaml.v3 v3.0.1
)
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/odvcencio/gotreesitter v0.16.0 h1:pg29HG6idOpmbh6BkPc6ncFP0C8rH9ISAzFVNdHm2ss=
github.com/odvcencio/gotreesitter v0.16.0/go.mod h1:ccYZsDUmAJQAtliLsNHT33F3X4AN7f/Z6JGiPNZoEzY=
github.com/odvcencio/gotreesitter v0.48.0 h1:Gr9XVbGs/1yqAsLZWp2zmW91XM4YEMJsWiZa35fm4R8=
github.com/odvcencio/gotreesitter v0.48.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down
Loading