Skip to content

Commit d4d0503

Browse files
claudeclaude
committed
core: speed up the analysis catalog with prepared statements
The core catalog backs every lookup with a query against its in-memory SQLite database, and handed the raw *sql.DB to the generated queries. Go's database/sql prepares and discards a statement per call, so SQLite re-parsed and re-planned the same dozen queries thousands of times per run: profiling analyzer.Prepare put 65% of samples inside sqlite3Parser and 85% under database/sql dispatch, with the analyzer itself barely on the profile. - Prepare each catalog statement once and rebind it, for both the seed path and the analyzer's lookups. - Install dialect seeds in a single transaction, and drop journalling for a database that is rebuilt from scratch on every run. - Pin the pool to one connection. Each ":memory:" connection is its own empty database, so a second one would have seen a catalog with no tables in it — a latent bug under any concurrent use. On the analyzer side, resolving a column no longer allocates a candidate slice per reference, the scope holds the catalog's column list instead of copying it into a scope-local type, a target's field list is flattened once instead of three times, and parameters are tracked by value. Analyzing 500 GoogleSQL queries drops from 181ms to 82ms end to end, and 100 queries from 58ms to 34ms, with byte-identical output on GoogleSQL and ClickHouse. Microbenchmarks (internal/core/analyzer/bench_test.go): CatalogNew 6001532 ns/op (catalog + dialect seed) CatalogNew 2863319 ns/op Prepare 1579238 ns/op (5 statements, warm catalog) Prepare 398200 ns/op Lookup volume is unchanged at 13 catalog queries per statement, and analysis still allocates ~400 times per statement — nearly all of it database/sql scanning a row per lookup. Memoizing lookups removes both, but needs cache invalidation and a read-only contract on the slices the catalog returns; that is not worth it for an in-memory database, so it is left out. internal/core had no tests. catalog_test.go covers lookups across catalog mutations and concurrent reads against the pinned connection; TestCatalogQueryVolume records queries and allocations per statement and fails on a lookup added inside a loop. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Fp7UGoiFnUwK9ZkDBoGy7u
1 parent 673f56d commit d4d0503

13 files changed

Lines changed: 847 additions & 99 deletions

File tree

‎internal/core/analyzer/analyzer.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ func Prepare(cat *core.Catalog, stmt ast.Node) (core.PrepareResult, error) {
1313
}
1414
a := &analyzer{
1515
cat: cat,
16-
params: map[int]*core.Parameter{},
16+
params: map[int]core.Parameter{},
1717
}
1818
switch s := stmt.(type) {
1919
case *ast.SelectStmt:
@@ -46,7 +46,7 @@ type analyzer struct {
4646
cat *core.Catalog
4747
scope *scope
4848
columns []core.Column
49-
params map[int]*core.Parameter
49+
params map[int]core.Parameter
5050
command core.Command
5151
}
5252

@@ -58,7 +58,7 @@ func (a *analyzer) result() core.PrepareResult {
5858
}
5959
}
6060

61-
func orderedParams(m map[int]*core.Parameter) []core.Parameter {
61+
func orderedParams(m map[int]core.Parameter) []core.Parameter {
6262
if len(m) == 0 {
6363
return nil
6464
}
@@ -71,7 +71,7 @@ func orderedParams(m map[int]*core.Parameter) []core.Parameter {
7171
out := make([]core.Parameter, 0, len(m))
7272
for i := 1; i <= maxN; i++ {
7373
if p, ok := m[i]; ok {
74-
out = append(out, *p)
74+
out = append(out, p)
7575
}
7676
}
7777
return out
@@ -112,6 +112,7 @@ func (a *analyzer) analyzeSelect(s *ast.SelectStmt) error {
112112
if targets == nil {
113113
return fmt.Errorf("select: empty target list")
114114
}
115+
a.columns = make([]core.Column, 0, len(targets))
115116
for _, t := range targets {
116117
rt, ok := t.(*ast.ResTarget)
117118
if !ok {

‎internal/core/analyzer/bench_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package analyzer_test
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"github.com/sqlc-dev/sqlc/internal/config"
8+
"github.com/sqlc-dev/sqlc/internal/core"
9+
"github.com/sqlc-dev/sqlc/internal/core/analyzer"
10+
coreschema "github.com/sqlc-dev/sqlc/internal/core/schema"
11+
"github.com/sqlc-dev/sqlc/internal/engine/googlesql"
12+
"github.com/sqlc-dev/sqlc/internal/sql/ast"
13+
"github.com/sqlc-dev/sqlc/internal/sql/rewrite"
14+
"github.com/sqlc-dev/sqlc/internal/sql/validate"
15+
)
16+
17+
const benchSchema = `
18+
CREATE TABLE users (
19+
id INT64 NOT NULL,
20+
name STRING NOT NULL,
21+
bio STRING,
22+
) PRIMARY KEY (id);
23+
24+
CREATE TABLE posts (
25+
id INT64 NOT NULL,
26+
user_id INT64 NOT NULL,
27+
title STRING(255),
28+
created TIMESTAMP NOT NULL,
29+
) PRIMARY KEY (id);
30+
`
31+
32+
const benchQueries = `
33+
SELECT * FROM users;
34+
SELECT COUNT(*) AS total FROM users;
35+
SELECT u.name, p.* FROM users u JOIN posts p ON p.user_id = u.id WHERE u.name = @name;
36+
SELECT id, name, bio FROM users WHERE id = @id AND name = @name;
37+
SELECT p.title, p.created FROM posts p WHERE p.user_id = @uid AND p.title = @t;
38+
`
39+
40+
func parseAll(t testing.TB, src string) []ast.Node {
41+
p := googlesql.NewParser()
42+
stmts, err := p.Parse(strings.NewReader(src))
43+
if err != nil {
44+
t.Fatal(err)
45+
}
46+
out := make([]ast.Node, 0, len(stmts))
47+
for i := range stmts {
48+
out = append(out, stmts[i].Raw)
49+
}
50+
return out
51+
}
52+
53+
// parseQueries mirrors the compiler pipeline: parse, then rewrite named
54+
// parameters into ParamRef nodes before handing the statement to the analyzer.
55+
func parseQueries(t testing.TB, src string) []ast.Node {
56+
out := make([]ast.Node, 0)
57+
for _, n := range parseAll(t, src) {
58+
raw, ok := n.(*ast.RawStmt)
59+
if !ok {
60+
t.Fatalf("not a raw statement: %T", n)
61+
}
62+
numbers, dollar, err := validate.ParamRef(raw)
63+
if err != nil {
64+
t.Fatal(err)
65+
}
66+
rewritten, _, _ := rewrite.NamedParameters(config.EngineGoogleSQL, raw, numbers, dollar)
67+
out = append(out, rewritten)
68+
}
69+
return out
70+
}
71+
72+
func newBenchCatalog(t testing.TB) (*core.Catalog, []ast.Node) {
73+
cat, err := core.New(googlesql.Dialect())
74+
if err != nil {
75+
t.Fatal(err)
76+
}
77+
for _, n := range parseAll(t, benchSchema) {
78+
if err := coreschema.Apply(cat, n); err != nil {
79+
t.Fatal(err)
80+
}
81+
}
82+
return cat, parseQueries(t, benchQueries)
83+
}
84+
85+
// BenchmarkCatalogNew measures per-compile catalog setup: open the SQLite
86+
// catalog, install the schema, and run the dialect seed.
87+
func BenchmarkCatalogNew(b *testing.B) {
88+
for b.Loop() {
89+
cat, err := core.New(googlesql.Dialect())
90+
if err != nil {
91+
b.Fatal(err)
92+
}
93+
cat.Close()
94+
}
95+
}
96+
97+
// BenchmarkApplySchema measures loading user DDL into the catalog.
98+
func BenchmarkApplySchema(b *testing.B) {
99+
stmts := parseAll(b, benchSchema)
100+
for b.Loop() {
101+
cat, err := core.New(googlesql.Dialect())
102+
if err != nil {
103+
b.Fatal(err)
104+
}
105+
for _, n := range stmts {
106+
if err := coreschema.Apply(cat, n); err != nil {
107+
b.Fatal(err)
108+
}
109+
}
110+
cat.Close()
111+
}
112+
}
113+
114+
// BenchmarkPrepare measures steady-state query analysis against a warm catalog.
115+
func BenchmarkPrepare(b *testing.B) {
116+
cat, queries := newBenchCatalog(b)
117+
defer cat.Close()
118+
for b.Loop() {
119+
for _, q := range queries {
120+
if _, err := analyzer.Prepare(cat, q); err != nil {
121+
b.Fatal(err)
122+
}
123+
}
124+
}
125+
}
126+
127+
// TestPrepareFixtures keeps the benchmark inputs honest: a fixture that stops
128+
// analyzing cleanly would otherwise turn into a silently faster benchmark.
129+
func TestPrepareFixtures(t *testing.T) {
130+
cat, queries := newBenchCatalog(t)
131+
defer cat.Close()
132+
for _, q := range queries {
133+
if _, err := analyzer.Prepare(cat, q); err != nil {
134+
t.Fatal(err)
135+
}
136+
}
137+
}

‎internal/core/analyzer/dml.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package analyzer
33
import (
44
"fmt"
55

6+
"github.com/sqlc-dev/sqlc/internal/core"
67
"github.com/sqlc-dev/sqlc/internal/sql/ast"
78
)
89

@@ -89,12 +90,12 @@ func (a *analyzer) relationScope(relations, extra *ast.List) (*scope, error) {
8990
return sc, nil
9091
}
9192

92-
func insertTargets(rel scopeRel, cols *ast.List) ([]scopeCol, error) {
93+
func insertTargets(rel scopeRel, cols *ast.List) ([]core.ClassColumn, error) {
9394
items := listItems(cols)
9495
if len(items) == 0 {
9596
return rel.cols, nil
9697
}
97-
out := make([]scopeCol, 0, len(items))
98+
out := make([]core.ClassColumn, 0, len(items))
9899
for _, item := range items {
99100
rt, ok := item.(*ast.ResTarget)
100101
if !ok || rt.Name == nil {
@@ -109,7 +110,7 @@ func insertTargets(rel scopeRel, cols *ast.List) ([]scopeCol, error) {
109110
return out, nil
110111
}
111112

112-
func (a *analyzer) bindInsertValues(n ast.Node, rel scopeRel, targets []scopeCol) error {
113+
func (a *analyzer) bindInsertValues(n ast.Node, rel scopeRel, targets []core.ClassColumn) error {
113114
if n == nil {
114115
return nil
115116
}
@@ -126,7 +127,7 @@ func (a *analyzer) bindInsertValues(n ast.Node, rel scopeRel, targets []scopeCol
126127
continue
127128
}
128129
for i, v := range values.Items {
129-
var target *scopeCol
130+
var target *core.ClassColumn
130131
if i < len(targets) {
131132
target = &targets[i]
132133
}
@@ -138,7 +139,7 @@ func (a *analyzer) bindInsertValues(n ast.Node, rel scopeRel, targets []scopeCol
138139
return nil
139140
}
140141

141-
func (a *analyzer) bindValue(rel scopeRel, target *scopeCol, v ast.Node) error {
142+
func (a *analyzer) bindValue(rel scopeRel, target *core.ClassColumn, v ast.Node) error {
142143
if target != nil {
143144
switch value := v.(type) {
144145
case *ast.ParamRef:
@@ -165,21 +166,21 @@ func (a *analyzer) projectReturning(l *ast.List) error {
165166
return nil
166167
}
167168

168-
func findColumn(rel scopeRel, name string) (scopeCol, bool) {
169+
func findColumn(rel scopeRel, name string) (core.ClassColumn, bool) {
169170
for _, col := range rel.cols {
170-
if col.name == name {
171+
if col.Name == name {
171172
return col, true
172173
}
173174
}
174-
return scopeCol{}, false
175+
return core.ClassColumn{}, false
175176
}
176177

177-
func columnType(rel scopeRel, col scopeCol) exprType {
178+
func columnType(rel scopeRel, col core.ClassColumn) exprType {
178179
return exprType{
179-
typeOID: col.typeOID,
180-
nullable: !col.notNull,
180+
typeOID: col.TypeOID,
181+
nullable: !col.NotNull,
181182
sourceClassOID: rel.classOID,
182-
sourceAttributeOID: col.attOID,
183+
sourceAttributeOID: col.AttOID,
183184
sourceTableAlias: rel.alias,
184185
}
185186
}

‎internal/core/analyzer/expr.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,10 @@ func (a *analyzer) typeColumnRef(c *ast.ColumnRef) (exprType, error) {
113113
return exprType{}, fmt.Errorf("unknown column %q", column)
114114
}
115115
return exprType{
116-
typeOID: col.typeOID,
117-
nullable: !col.notNull,
116+
typeOID: col.TypeOID,
117+
nullable: !col.NotNull,
118118
sourceClassOID: rel.classOID,
119-
sourceAttributeOID: col.attOID,
119+
sourceAttributeOID: col.AttOID,
120120
sourceTableAlias: rel.alias,
121121
}, nil
122122
}
@@ -141,7 +141,7 @@ func flattenFields(fields *ast.List) []string {
141141
func (a *analyzer) typeParamRef(p *ast.ParamRef) (exprType, error) {
142142
cur, ok := a.params[p.Number]
143143
if !ok {
144-
cur = &core.Parameter{Number: p.Number}
144+
cur = core.Parameter{Number: p.Number}
145145
a.params[p.Number] = cur
146146
}
147147
return exprType{typeOID: cur.TypeOID, nullable: !cur.NotNull}, nil
@@ -150,8 +150,7 @@ func (a *analyzer) typeParamRef(p *ast.ParamRef) (exprType, error) {
150150
func (a *analyzer) inferParam(number int, t exprType) {
151151
cur, ok := a.params[number]
152152
if !ok {
153-
cur = &core.Parameter{Number: number}
154-
a.params[number] = cur
153+
cur = core.Parameter{Number: number}
155154
}
156155
if cur.TypeOID == 0 && t.typeOID != 0 {
157156
cur.TypeOID = t.typeOID
@@ -171,6 +170,7 @@ func (a *analyzer) inferParam(number int, t exprType) {
171170
}
172171
}
173172
}
173+
a.params[number] = cur
174174
}
175175

176176
func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) {

0 commit comments

Comments
 (0)