From ae3e7fd6b401cf53fa2b21440ac036bdf74bc789 Mon Sep 17 00:00:00 2001 From: Harsh Kapse Date: Sun, 2 Aug 2026 10:41:45 +0530 Subject: [PATCH 1/5] deps(treesitter): upgrade gotreesitter from v0.16.0 to v0.48.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9a297f3..e023afb 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 github.com/mattn/go-isatty v0.0.20 - github.com/odvcencio/gotreesitter v0.16.0 + github.com/odvcencio/gotreesitter v0.48.0 github.com/spf13/cobra v1.10.2 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 2ceef3c..e47f433 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/odvcencio/gotreesitter v0.16.0 h1:pg29HG6idOpmbh6BkPc6ncFP0C8rH9ISAzFVNdHm2ss= -github.com/odvcencio/gotreesitter v0.16.0/go.mod h1:ccYZsDUmAJQAtliLsNHT33F3X4AN7f/Z6JGiPNZoEzY= +github.com/odvcencio/gotreesitter v0.48.0 h1:Gr9XVbGs/1yqAsLZWp2zmW91XM4YEMJsWiZa35fm4R8= +github.com/odvcencio/gotreesitter v0.48.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= From 3b8c0afd5bc700d1e771e858c79c62dd9933a3db Mon Sep 17 00:00:00 2001 From: Harsh Kapse Date: Sun, 2 Aug 2026 11:03:34 +0530 Subject: [PATCH 2/5] ci: format benchmark A/B summaries and enhance coverage reporting --- .github/scripts/format_benchstat.py | 170 ++++++++++++++++++++++++++++ .github/workflows/ci.yml | 28 ++--- 2 files changed, 185 insertions(+), 13 deletions(-) create mode 100755 .github/scripts/format_benchstat.py diff --git a/.github/scripts/format_benchstat.py b/.github/scripts/format_benchstat.py new file mode 100755 index 0000000..44288fe --- /dev/null +++ b/.github/scripts/format_benchstat.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Formats raw Go benchstat text output into a clean, executive GitHub Step Summary Markdown report. +""" +import sys +import re + +def parse_val_unit(s): + """Parses value string like '59.30m', '7.601Mi', '18.84k' into a numeric float.""" + s = s.strip() + m = re.match(r'^([0-9.]+)([a-zA-Z%]*)$', s) + if not m: + return 0.0 + val_str, unit = m.groups() + val = float(val_str) + multiplier = 1.0 + if unit == 'k': multiplier = 1e3 + elif unit in ('M', 'Mi'): multiplier = 1e6 + elif unit in ('G', 'Gi'): multiplier = 1e9 + elif unit == 'm': multiplier = 1e-3 + elif unit in ('u', 'ยต'): multiplier = 1e-6 + elif unit == 'n': multiplier = 1e-9 + return val * multiplier + +def compute_pct(old_s, new_s): + old_v = parse_val_unit(old_s) + new_v = parse_val_unit(new_s) + if old_v == 0: + return 0.0 + return ((new_v - old_v) / old_v) * 100.0 + +def format_benchstat(text): + lines = text.splitlines() + current_metric = None + geomeans = {} + fixtures = {} # fix_name -> {'time': (old, new), 'mem': (old, new), 'allocs': (old, new)} + + for line in lines: + line_str = line.strip() + if 'sec/op' in line_str and 'vs base' in line_str: + current_metric = 'time' + elif 'B/op' in line_str and 'vs base' in line_str: + current_metric = 'mem' + elif 'allocs/op' in line_str and 'vs base' in line_str: + current_metric = 'allocs' + elif line_str.startswith('geomean'): + parts = line_str.split() + if len(parts) >= 3 and current_metric: + delta = parts[3] if len(parts) >= 4 else "~" + geomeans[current_metric] = (parts[1], parts[2], delta) + elif line_str.startswith('Pipeline/'): + parts = line_str.split() + if len(parts) >= 3 and current_metric: + fixture_raw = parts[0] + fix_name = fixture_raw.replace('Pipeline/', '') + if '-' in fix_name: + fix_name = fix_name.rsplit('-', 1)[0] + + vals = [p for p in parts[1:] if re.match(r'^[0-9.]+[a-zA-Z]*$', p)] + if len(vals) >= 2: + old_v, new_v = vals[0], vals[1] + if fix_name not in fixtures: + fixtures[fix_name] = {} + fixtures[fix_name][current_metric] = (old_v, new_v) + + # Build Markdown Output + md = [] + md.append("### ๐Ÿ“Š Benchmark A/B Comparison Report") + md.append("") + md.append("#### ๐Ÿ“ˆ Key Metrics Summary (Geometric Mean)") + md.append("| Metric | Baseline | Current | Delta | Status |") + md.append("| :--- | :---: | :---: | :---: | :---: |") + + metric_meta = { + 'time': ('Execution Time', 'sec/op'), + 'mem': ('Memory Allocated', 'B/op'), + 'allocs': ('Allocations', 'allocs/op') + } + + for key in ['time', 'mem', 'allocs']: + if key in geomeans: + old_g, new_g, _ = geomeans[key] + pct = compute_pct(old_g, new_g) + name, _ = metric_meta[key] + + if abs(pct) < 0.5: + status = "โž– Neutral" + badge = f"`{pct:+.2f}%`" + elif pct < 0: + status = "๐ŸŸข Faster" if key == 'time' else "๐ŸŸข Reduced" + badge = f"**`{pct:+.2f}%`**" + else: + status = "๐Ÿ”ด Slower" if key == 'time' else "๐ŸŸก Increased" + badge = f"**`{pct:+.2f}%`**" + + md.append(f"| **{name}** | `{old_g}` | `{new_g}` | {badge} | {status} |") + + md.append("") + + # Sort changes by time delta + changes = [] + for fix_name, data in fixtures.items(): + if 'time' in data: + old_t, new_t = data['time'] + pct = compute_pct(old_t, new_t) + changes.append((fix_name, pct, old_t, new_t, data.get('mem', ('-', '-')))) + + changes.sort(key=lambda x: x[1]) + + speedups = [c for c in changes if c[1] <= -10.0] + regressions = [c for c in changes if c[1] >= 10.0] + + if speedups or regressions: + md.append("#### โšก Notable Performance Shifts (โ‰ฅ 10% change)") + md.append("| Fixture | Baseline Time | Current Time | Time Delta | Memory Delta |") + md.append("| :--- | :---: | :---: | :---: | :---: |") + + for fix_name, pct, old_t, new_t, mem_tuple in speedups + regressions: + icon = "๐Ÿš€" if pct < 0 else "โš ๏ธ" + old_m, new_m = mem_tuple + mem_pct_str = "-" + if old_m != '-' and new_m != '-': + m_pct = compute_pct(old_m, new_m) + mem_pct_str = f"`{m_pct:+.1f}%`" + md.append(f"| {icon} `{fix_name}` | `{old_t}` | `{new_t}` | **`{pct:+.1f}%`** | {mem_pct_str} |") + md.append("") + + # Collapsible Table for All Fixtures + md.append("
") + md.append(f"๐Ÿ” Full Breakdown ({len(fixtures)} Fixtures)") + md.append("") + md.append("| Fixture | Baseline Time | Current Time | Time Delta | Baseline Mem | Current Mem | Mem Delta |") + md.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: |") + + for fix_name, data in sorted(fixtures.items()): + old_t, new_t = data.get('time', ('-', '-')) + old_m, new_m = data.get('mem', ('-', '-')) + + t_pct_str = "`~`" + if old_t != '-' and new_t != '-': + t_pct = compute_pct(old_t, new_t) + t_pct_str = f"`{t_pct:+.1f}%`" if abs(t_pct) >= 0.5 else "`~`" + + m_pct_str = "`~`" + if old_m != '-' and new_m != '-': + m_pct = compute_pct(old_m, new_m) + m_pct_str = f"`{m_pct:+.1f}%`" if abs(m_pct) >= 0.5 else "`~`" + + md.append(f"| `{fix_name}` | `{old_t}` | `{new_t}` | {t_pct_str} | `{old_m}` | `{new_m}` | {m_pct_str} |") + + md.append("") + md.append("
") + md.append("") + md.append("
") + md.append("๐Ÿ“„ Raw benchstat Console Output") + md.append("") + md.append("```") + md.append(text.strip()) + md.append("```") + md.append("
") + + return "\n".join(md) + +if __name__ == '__main__': + if len(sys.argv) > 1: + with open(sys.argv[1], 'r') as f: + content = f.read() + else: + content = sys.stdin.read() + print(format_benchstat(content)) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2598e0..7894a03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,16 +61,24 @@ jobs: -coverprofile=coverage.out -covermode=atomic -coverpkg=./... ./... - - name: Print coverage summary + - name: Print & publish coverage summary if: matrix.os == 'ubuntu-latest' - run: go tool cover -func=coverage.out | tail -1 + run: | + go tool cover -func=coverage.out | tail -1 + echo "### ๐Ÿงช Code Coverage" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + go tool cover -func=coverage.out | tail -15 >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + go tool cover -html=coverage.out -o coverage.html - name: Upload coverage report if: matrix.os == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: name: coverage-report - path: coverage.out + path: | + coverage.out + coverage.html - name: Publish Test Report uses: mikepenz/action-junit-report@v4 @@ -79,10 +87,6 @@ jobs: report_paths: '**/unit-tests.xml' check_name: 'Test Report (${{ matrix.os }})' - - name: Benchmark smoke test - if: matrix.os == 'ubuntu-latest' - run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=1 -benchtime=1x - - name: Benchmark A/B comparison (benchstat & GITHUB_STEP_SUMMARY) if: matrix.os == 'ubuntu-latest' run: | @@ -96,9 +100,7 @@ jobs: echo "==> Running benchmarks on baseline ($BASE_REF)..." go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true - # Render Markdown comparison table for GitHub Actions Summary Tab - echo "### ๐Ÿ“Š Benchmark A/B Comparison vs Baseline" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - benchstat bench_old.txt bench_new.txt >> $GITHUB_STEP_SUMMARY || true - echo '```' >> $GITHUB_STEP_SUMMARY - rm -f bench_old.txt bench_new.txt + # Render formatted Markdown comparison report for GitHub Actions Summary Tab + benchstat bench_old.txt bench_new.txt > benchstat_raw.txt || true + python3 .github/scripts/format_benchstat.py benchstat_raw.txt >> $GITHUB_STEP_SUMMARY || cat benchstat_raw.txt >> $GITHUB_STEP_SUMMARY + rm -f bench_old.txt bench_new.txt benchstat_raw.txt From 0a4b312683148f72c1fc589853c5cd3793e891a1 Mon Sep 17 00:00:00 2001 From: Harsh Kapse Date: Sun, 2 Aug 2026 11:07:33 +0530 Subject: [PATCH 3/5] ci: separate benchmark A/B comparison into a dedicated job --- .github/workflows/ci.yml | 63 +++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7894a03..526acdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,20 +87,61 @@ jobs: report_paths: '**/unit-tests.xml' check_name: 'Test Report (${{ matrix.os }})' - - name: Benchmark A/B comparison (benchstat & GITHUB_STEP_SUMMARY) - if: matrix.os == 'ubuntu-latest' + benchmark: + name: Benchmark A/B Comparison + needs: test + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check performance-critical file changes + id: filter + run: | + BASE_REF="${{ github.event.pull_request.base.sha || 'HEAD~1' }}" + CHANGED=$(git diff --name-only $BASE_REF HEAD || true) + echo "Changed files:" + echo "$CHANGED" + if echo "$CHANGED" | grep -qE '^(internal/engine/|internal/treesitter/|internal/actions/|tests/integration/|go\.mod|go\.sum)'; then + echo "::notice::Performance-critical files changed. Running A/B benchmarks..." + echo "run_benchmarks=true" >> $GITHUB_OUTPUT + else + echo "::notice::No performance-critical files changed. Skipping benchmarks." + echo "run_benchmarks=false" >> $GITHUB_OUTPUT + fi + + - name: Set up Go + if: steps.filter.outputs.run_benchmarks == 'true' + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + cache-dependency-path: 'go.sum' + + - name: Install benchstat + if: steps.filter.outputs.run_benchmarks == 'true' + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Run benchmarks on current HEAD + if: steps.filter.outputs.run_benchmarks == 'true' + run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_new.txt + + - name: Fetch baseline branch + if: steps.filter.outputs.run_benchmarks == 'true' run: | - go install golang.org/x/perf/cmd/benchstat@latest - echo "==> Running benchmarks on current HEAD..." - go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_new.txt - echo "==> Fetching baseline..." - git fetch origin ${{ github.event.pull_request.base.ref || 'main' }} --depth=2 || true BASE_REF="${{ github.event.pull_request.base.sha || 'HEAD~1' }}" git checkout $BASE_REF || true - echo "==> Running benchmarks on baseline ($BASE_REF)..." - go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true - - # Render formatted Markdown comparison report for GitHub Actions Summary Tab + + - name: Run benchmarks on baseline + if: steps.filter.outputs.run_benchmarks == 'true' + run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true + + - name: Render Benchmark A/B Summary + if: steps.filter.outputs.run_benchmarks == 'true' + run: | benchstat bench_old.txt bench_new.txt > benchstat_raw.txt || true python3 .github/scripts/format_benchstat.py benchstat_raw.txt >> $GITHUB_STEP_SUMMARY || cat benchstat_raw.txt >> $GITHUB_STEP_SUMMARY rm -f bench_old.txt bench_new.txt benchstat_raw.txt From 3e1ecb1ee90180d45ba213f667a91e326157040e Mon Sep 17 00:00:00 2001 From: Harsh Kapse Date: Sun, 2 Aug 2026 11:26:55 +0530 Subject: [PATCH 4/5] fix(ci): restore PR HEAD checkout before running format_benchstat.py --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 526acdd..1d2ae5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,10 @@ jobs: - name: Run benchmarks on baseline if: steps.filter.outputs.run_benchmarks == 'true' - run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true + run: | + go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true + echo "==> Restoring PR HEAD commit to access scripts..." + git checkout ${{ github.sha }} || git checkout - || true - name: Render Benchmark A/B Summary if: steps.filter.outputs.run_benchmarks == 'true' From 2157354fd70e5dc6b957389361c23860ed3697e4 Mon Sep 17 00:00:00 2001 From: Harsh Kapse Date: Sun, 2 Aug 2026 11:36:34 +0530 Subject: [PATCH 5/5] ci: refactor into lint, test-linux, test-cross, and benchmark jobs --- .github/workflows/ci.yml | 70 ++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d2ae5a..ac1c2ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,13 +17,9 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - test: - name: Test & Build - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + lint: + name: Lint & Format Check + runs-on: ubuntu-latest steps: - name: Checkout code @@ -37,32 +33,40 @@ jobs: cache-dependency-path: 'go.sum' - name: Lint (golangci-lint) - if: matrix.os == 'ubuntu-latest' uses: golangci/golangci-lint-action@v9 with: version: v2.12 args: --timeout=5m - name: Verify formatting (golangci-lint fmt) - if: matrix.os == 'ubuntu-latest' run: golangci-lint fmt --diff ./... + test-linux: + name: Test & Coverage (Linux) + needs: lint + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + cache-dependency-path: 'go.sum' + - name: Install gotestsum run: go install gotest.tools/gotestsum@latest - - name: Run test suite - if: matrix.os != 'ubuntu-latest' - run: gotestsum --format pkgname --junitfile unit-tests.xml -- ./... - - name: Run test suite with coverage - if: matrix.os == 'ubuntu-latest' run: >- gotestsum --format pkgname --junitfile unit-tests.xml -- -coverprofile=coverage.out -covermode=atomic -coverpkg=./... ./... - name: Print & publish coverage summary - if: matrix.os == 'ubuntu-latest' run: | go tool cover -func=coverage.out | tail -1 echo "### ๐Ÿงช Code Coverage" >> $GITHUB_STEP_SUMMARY @@ -72,7 +76,6 @@ jobs: go tool cover -html=coverage.out -o coverage.html - name: Upload coverage report - if: matrix.os == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: name: coverage-report @@ -80,6 +83,39 @@ jobs: coverage.out coverage.html + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: always() + with: + report_paths: '**/unit-tests.xml' + check_name: 'Test Report (ubuntu-latest)' + + test-cross: + name: Cross-Platform Test + needs: lint + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + cache-dependency-path: 'go.sum' + + - name: Install gotestsum + run: go install gotest.tools/gotestsum@latest + + - name: Run test suite + run: gotestsum --format pkgname --junitfile unit-tests.xml -- ./... + - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: always() @@ -89,7 +125,7 @@ jobs: benchmark: name: Benchmark A/B Comparison - needs: test + needs: [lint, test-linux] runs-on: ubuntu-latest steps: