diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 562d8f9..a2598e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,11 +51,54 @@ jobs: run: go install gotest.tools/gotestsum@latest - name: Run test suite + if: matrix.os != 'ubuntu-latest' run: gotestsum --format pkgname --junitfile unit-tests.xml -- ./... + - name: Run test suite with coverage + if: matrix.os == 'ubuntu-latest' + run: >- + gotestsum --format pkgname --junitfile unit-tests.xml -- + -coverprofile=coverage.out -covermode=atomic -coverpkg=./... + ./... + + - name: Print coverage summary + if: matrix.os == 'ubuntu-latest' + run: go tool cover -func=coverage.out | tail -1 + + - name: Upload coverage report + if: matrix.os == 'ubuntu-latest' + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: coverage.out + - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: always() with: report_paths: '**/unit-tests.xml' check_name: 'Test Report (${{ matrix.os }})' + + - name: Benchmark smoke test + if: matrix.os == 'ubuntu-latest' + run: go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=1 -benchtime=1x + + - name: Benchmark A/B comparison (benchstat & GITHUB_STEP_SUMMARY) + if: matrix.os == 'ubuntu-latest' + run: | + go install golang.org/x/perf/cmd/benchstat@latest + echo "==> Running benchmarks on current HEAD..." + go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_new.txt + echo "==> Fetching baseline..." + git fetch origin ${{ github.event.pull_request.base.ref || 'main' }} --depth=2 || true + BASE_REF="${{ github.event.pull_request.base.sha || 'HEAD~1' }}" + git checkout $BASE_REF || true + echo "==> Running benchmarks on baseline ($BASE_REF)..." + go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$ -count=3 > bench_old.txt || true + + # Render Markdown comparison table for GitHub Actions Summary Tab + echo "### 📊 Benchmark A/B Comparison vs Baseline" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + benchstat bench_old.txt bench_new.txt >> $GITHUB_STEP_SUMMARY || true + echo '```' >> $GITHUB_STEP_SUMMARY + rm -f bench_old.txt bench_new.txt diff --git a/Makefile b/Makefile index 3c72ef7..1b63554 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: test test-unit test-integration test-e2e lint fmt coverage test-update +.PHONY: test test-unit test-integration test-e2e lint fmt coverage test-update bench bench-short -test: fmt lint test-unit test-integration test-e2e ## Run everything +test: lint test-unit test-integration test-e2e ## Run everything test-unit: ## Unit tests only go test ./internal/... -count=1 @@ -14,14 +14,28 @@ test-e2e: ## E2E CLI tests test-update: ## Regenerate golden files go test ./tests/integration/ -v -update +bench: ## Run all benchmarks + go test ./tests/integration/ -bench=. -benchmem -run=^$$ -count=1 + +bench-short: ## Quick benchmark smoke test (single iteration) + go test ./tests/integration/ -bench=BenchmarkPipeline -benchmem -run=^$$ -count=1 -benchtime=1x + coverage: ## Coverage report go test ./internal/... -coverprofile=coverage.out -covermode=atomic go tool cover -func=coverage.out | tail -1 @echo "" @echo "Full report: go tool cover -html=coverage.out -o coverage.html" -lint: +lint: ## Run linter and check code formatting + @DIFF=$$(golangci-lint fmt --diff ./...); \ + if [ -n "$$DIFF" ]; then \ + echo "Formatting errors found:"; \ + echo "$$DIFF"; \ + echo ""; \ + echo "Run 'golangci-lint fmt ./...' or 'make fmt' to fix formatting."; \ + exit 1; \ + fi golangci-lint run ./... -fmt: +fmt: ## Format code automatically golangci-lint fmt ./... diff --git a/tests/integration/pipeline_bench_test.go b/tests/integration/pipeline_bench_test.go new file mode 100644 index 0000000..44fa6e2 --- /dev/null +++ b/tests/integration/pipeline_bench_test.go @@ -0,0 +1,114 @@ +package integration + +import ( + "testing" + + "github.com/HarshK97/diffmantic/internal/actions" + "github.com/HarshK97/diffmantic/internal/engine" + "github.com/HarshK97/diffmantic/internal/postprocess" + "github.com/HarshK97/diffmantic/internal/serialize" +) + +// Run the full pipeline (parse -> match -> edit script -> postprocess -> serialize) across all fixtures. +func BenchmarkPipeline(b *testing.B) { + for _, name := range allFixtures(b) { + f := loadFixture(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + astA := mustParse(b, f.OldSrc, f.OldPath) + astB := mustParse(b, f.NewSrc, f.NewPath) + + result := engine.Match(astA, astB) + es := actions.GenerateEditScript(astA, astB, result.Mappings) + es = postprocess.Run(es, result.Mappings, astA, astB) + + if _, err := serialize.Marshal(es, result.Mappings, astA, astB, f.OldSrc, f.NewSrc); err != nil { + b.Fatalf("serializing: %v", err) + } + } + }) + } +} + +// Time tree-sitter parsing for old and new files. +func BenchmarkParse(b *testing.B) { + for _, name := range allFixtures(b) { + f := loadFixture(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + mustParse(b, f.OldSrc, f.OldPath) + mustParse(b, f.NewSrc, f.NewPath) + } + }) + } +} + +// Time AST matching. We re-parse on each iteration because Match mutates internal state. +func BenchmarkMatch(b *testing.B) { + for _, name := range allFixtures(b) { + f := loadFixture(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + astA := mustParse(b, f.OldSrc, f.OldPath) + astB := mustParse(b, f.NewSrc, f.NewPath) + engine.Match(astA, astB) + } + }) + } +} + +// Time edit script generation (Chawathe algorithm). +func BenchmarkEditScript(b *testing.B) { + for _, name := range allFixtures(b) { + f := loadFixture(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + astA := mustParse(b, f.OldSrc, f.OldPath) + astB := mustParse(b, f.NewSrc, f.NewPath) + result := engine.Match(astA, astB) + actions.GenerateEditScript(astA, astB, result.Mappings) + } + }) + } +} + +// Time postprocessing (collapsing, punctuation filtering, and action grouping). +func BenchmarkPostprocess(b *testing.B) { + for _, name := range allFixtures(b) { + f := loadFixture(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + astA := mustParse(b, f.OldSrc, f.OldPath) + astB := mustParse(b, f.NewSrc, f.NewPath) + result := engine.Match(astA, astB) + es := actions.GenerateEditScript(astA, astB, result.Mappings) + postprocess.Run(es, result.Mappings, astA, astB) + } + }) + } +} + +// Time JSON envelope creation and marshalling. +func BenchmarkSerialize(b *testing.B) { + for _, name := range allFixtures(b) { + f := loadFixture(b, name) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + astA := mustParse(b, f.OldSrc, f.OldPath) + astB := mustParse(b, f.NewSrc, f.NewPath) + result := engine.Match(astA, astB) + es := actions.GenerateEditScript(astA, astB, result.Mappings) + es = postprocess.Run(es, result.Mappings, astA, astB) + if _, err := serialize.Marshal(es, result.Mappings, astA, astB, f.OldSrc, f.NewSrc); err != nil { + b.Fatalf("serializing: %v", err) + } + } + }) + } +} diff --git a/tests/integration/pipeline_test.go b/tests/integration/pipeline_test.go index b83f459..b17c578 100644 --- a/tests/integration/pipeline_test.go +++ b/tests/integration/pipeline_test.go @@ -27,15 +27,15 @@ import ( var update = flag.Bool("update", false, "update the golden files") // testdataDir returns the absolute path to tests/testdata/. -func testdataDir(t *testing.T) string { - t.Helper() +func testdataDir(tb testing.TB) string { + tb.Helper() // pipeline_test.go is two levels below repo root; testdata is in tests/ dir, err := filepath.Abs(filepath.Join("..", "testdata")) if err != nil { - t.Fatalf("resolving testdata dir: %v", err) + tb.Fatalf("resolving testdata dir: %v", err) } if _, err := os.Stat(dir); os.IsNotExist(err) { - t.Fatalf("testdata dir does not exist: %s", dir) + tb.Fatalf("testdata dir does not exist: %s", dir) } return dir } @@ -50,13 +50,13 @@ type fixture struct { } // loadFixture reads the old and new source files for a fixture. -func loadFixture(t *testing.T, name string) fixture { - t.Helper() - dir := filepath.Join(testdataDir(t), name) +func loadFixture(tb testing.TB, name string) fixture { + tb.Helper() + dir := filepath.Join(testdataDir(tb), name) entries, err := os.ReadDir(dir) if err != nil { - t.Fatalf("reading fixture dir %s: %v", name, err) + tb.Fatalf("reading fixture dir %s: %v", name, err) } var oldPath, newPath string @@ -69,16 +69,16 @@ func loadFixture(t *testing.T, name string) fixture { } } if oldPath == "" || newPath == "" { - t.Fatalf("fixture %s: missing old.* or new.* file", name) + tb.Fatalf("fixture %s: missing old.* or new.* file", name) } oldSrc, err := os.ReadFile(oldPath) if err != nil { - t.Fatalf("reading %s: %v", oldPath, err) + tb.Fatalf("reading %s: %v", oldPath, err) } newSrc, err := os.ReadFile(newPath) if err != nil { - t.Fatalf("reading %s: %v", newPath, err) + tb.Fatalf("reading %s: %v", newPath, err) } return fixture{ @@ -90,6 +90,15 @@ func loadFixture(t *testing.T, name string) fixture { } } +func mustParse(tb testing.TB, src []byte, path string) *treesitter.ASTNode { + tb.Helper() + ast, err := treesitter.Parse(src, path) + if err != nil { + tb.Fatalf("parsing %s: %v", path, err) + } + return ast +} + // pipelineResult holds the output of a full engine run. type pipelineResult struct { AstA *treesitter.ASTNode @@ -131,12 +140,12 @@ func runPipeline(t *testing.T, f fixture) pipelineResult { } // allFixtures returns the names of all fixture directories in testdata/. -func allFixtures(t *testing.T) []string { - t.Helper() - dir := testdataDir(t) +func allFixtures(tb testing.TB) []string { + tb.Helper() + dir := testdataDir(tb) entries, err := os.ReadDir(dir) if err != nil { - t.Fatalf("reading testdata dir: %v", err) + tb.Fatalf("reading testdata dir: %v", err) } var names []string for _, e := range entries {