Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions apps/cli-go/internal/db/diff/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ func Run(ctx context.Context, schema []string, file string, config pgconn.Config
}

func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {
if schemas := utils.Config.Db.Migrations.SchemaPaths; len(schemas) > 0 {
return schemas.SQLFiles(afero.NewIOFS(fsys))
}
// When pg-delta is enabled, declarative path is the source of truth (config or default).
if utils.IsPgDeltaEnabled() {
declDir := utils.GetDeclarativeDir()
Expand All @@ -69,9 +72,6 @@ func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {
return declared, nil
}
}
if schemas := utils.Config.Db.Migrations.SchemaPaths; len(schemas) > 0 {
return schemas.Files(afero.NewIOFS(fsys))
}
if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil {
return nil, errors.Errorf("failed to check schemas: %w", err)
} else if !exists {
Expand All @@ -95,6 +95,24 @@ func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {
return declared, nil
}

func shouldApplyDeclarativeWithPgDelta(usePgDelta bool) bool {
if !usePgDelta {
return false
}
schemas := utils.Config.Db.Migrations.SchemaPaths
if len(schemas) == 0 {
return true
}
if len(schemas) != 1 {
return false
}
return cleanSchemaPath(schemas[0]) == cleanSchemaPath(utils.GetDeclarativeDir())
}

func cleanSchemaPath(path string) string {
return filepath.ToSlash(filepath.Clean(path))
}

// https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6
var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`)

Expand Down
160 changes: 160 additions & 0 deletions apps/cli-go/internal/db/diff/diff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/supabase/cli/internal/testing/helper"
"github.com/supabase/cli/internal/utils"
"github.com/supabase/cli/internal/utils/flags"
pkgconfig "github.com/supabase/cli/pkg/config"
"github.com/supabase/cli/pkg/migration"
"github.com/supabase/cli/pkg/pgtest"
)
Expand All @@ -36,6 +37,80 @@ var dbConfig = pgconn.Config{
Database: "postgres",
}

func TestLoadDeclaredSchemas(t *testing.T) {
t.Run("respects schema_paths order when pg-delta declarative dir exists", func(t *testing.T) {
originalConfig := utils.Config
t.Cleanup(func() { utils.Config = originalConfig })
utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{
"supabase/schemas/z_function.sql",
"supabase/schemas/a_table.sql",
}
utils.Config.Experimental.PgDelta = &pkgconfig.PgDeltaConfig{
Enabled: true,
DeclarativeSchemaPath: utils.SchemasDir,
}
fsys := afero.NewMemMapFs()
require.NoError(t, fsys.MkdirAll(utils.SchemasDir, 0755))
require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/a_table.sql", []byte("create table a();"), 0644))
require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/z_function.sql", []byte("create function z() returns void language sql as $$ select 1 $$;"), 0644))

declared, err := loadDeclaredSchemas(fsys)

require.NoError(t, err)
assert.Equal(t, []string{
"supabase/schemas/z_function.sql",
"supabase/schemas/a_table.sql",
}, declared)
})

t.Run("expands schema_paths directory entries deterministically", func(t *testing.T) {
originalConfig := utils.Config
t.Cleanup(func() { utils.Config = originalConfig })
utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{utils.DeclarativeDir}
fsys := afero.NewMemMapFs()
require.NoError(t, fsys.MkdirAll(filepath.Join(utils.DeclarativeDir, "nested"), 0755))
require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.DeclarativeDir, "nested", "b.sql"), []byte("select 2;"), 0644))
require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.DeclarativeDir, "a.sql"), []byte("select 1;"), 0644))

declared, err := loadDeclaredSchemas(fsys)

require.NoError(t, err)
assert.Equal(t, []string{
filepath.Join(utils.DeclarativeDir, "a.sql"),
filepath.Join(utils.DeclarativeDir, "nested", "b.sql"),
}, declared)
})
}

func TestShouldApplyDeclarativeWithPgDelta(t *testing.T) {
t.Run("uses pg-delta declarative apply when no schema_paths override is configured", func(t *testing.T) {
originalConfig := utils.Config
t.Cleanup(func() { utils.Config = originalConfig })
utils.Config.Db.Migrations.SchemaPaths = nil

assert.True(t, shouldApplyDeclarativeWithPgDelta(true))
})

t.Run("uses pg-delta declarative apply when schema_paths points at the declarative dir", func(t *testing.T) {
originalConfig := utils.Config
t.Cleanup(func() { utils.Config = originalConfig })
utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{utils.DeclarativeDir + "/"}

assert.True(t, shouldApplyDeclarativeWithPgDelta(true))
})

t.Run("uses ordered migration apply for explicit schema_paths files", func(t *testing.T) {
originalConfig := utils.Config
t.Cleanup(func() { utils.Config = originalConfig })
utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{
"supabase/schemas/z_function.sql",
"supabase/schemas/a_table.sql",
}

assert.False(t, shouldApplyDeclarativeWithPgDelta(true))
})
}

func TestRun(t *testing.T) {
t.Run("runs migra diff", func(t *testing.T) {
// Setup in-memory fs
Expand Down Expand Up @@ -98,6 +173,91 @@ func TestRun(t *testing.T) {
assert.Equal(t, []byte(diff), contents)
})

t.Run("applies schema_paths in order before saving generated diff", func(t *testing.T) {
originalConfig := utils.Config
t.Cleanup(func() { utils.Config = originalConfig })
utils.Config.Db.MajorVersion = 14
utils.Config.Db.ShadowPort = 54320
utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{
"supabase/schemas/z_function.sql",
"supabase/schemas/a_table.sql",
}
utils.Config.Experimental.PgDelta = &pkgconfig.PgDeltaConfig{
Enabled: true,
DeclarativeSchemaPath: utils.SchemasDir,
}
utils.GlobalsSql = "create schema public"
utils.InitialSchemaPg14Sql = "create schema private"
functionSQL := "create function public.z_function() returns integer language sql as $$ select 1 $$"
tableSQL := "create table public.a_table (id integer default public.z_function())"
generated := functionSQL + ";\n" + tableSQL + ";\n"
fsys := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/a_table.sql", []byte(tableSQL), 0644))
require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/z_function.sql", []byte(functionSQL), 0644))
require.NoError(t, apitest.MockDocker(utils.Docker))
defer gock.OffAll()
apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.Config.Db.Image), "test-shadow-db")
gock.New(utils.Docker.DaemonHost()).
Get("/v" + utils.Docker.ClientVersion() + "/containers/test-shadow-db/json").
Reply(http.StatusOK).
JSON(container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{
State: &container.State{
Running: true,
Health: &container.Health{Status: types.Healthy},
},
}})
gock.New(utils.Docker.DaemonHost()).
Delete("/v" + utils.Docker.ClientVersion() + "/containers/test-shadow-db").
Reply(http.StatusOK)
shadowConn := pgtest.NewConn()
defer shadowConn.Close(t)
shadowConn.Query(utils.GlobalsSql).
Reply("CREATE SCHEMA").
Query(utils.InitialSchemaPg14Sql).
Reply("CREATE SCHEMA")
helper.MockApiPrivilegesRevoke(shadowConn).
Query(CREATE_TEMPLATE).
Reply("CREATE DATABASE")
declaredConn := pgtest.NewConn()
defer declaredConn.Close(t)
declaredConn.Query(functionSQL).
Reply("CREATE FUNCTION").
Query(tableSQL).
Reply("CREATE TABLE")
diffCalled := false
differ := func(_ context.Context, _, target pgconn.Config, schema []string, _ ...func(*pgx.ConnConfig)) (string, error) {
diffCalled = true
assert.Equal(t, "contrib_regression", target.Database)
assert.Equal(t, []string{"public"}, schema)
return generated, nil
}
localConfig := pgconn.Config{
Host: utils.Config.Hostname,
Port: utils.Config.Db.Port,
User: "postgres",
Password: utils.Config.Db.Password,
Database: "postgres",
}

err := Run(context.Background(), []string{"public"}, "ordered_schema", localConfig, differ, true, fsys, func(cc *pgx.ConnConfig) {
if cc.Database == "contrib_regression" {
declaredConn.Intercept(cc)
} else {
shadowConn.Intercept(cc)
}
})

require.NoError(t, err)
assert.True(t, diffCalled)
assert.Empty(t, apitest.ListUnmatchedRequests())
files, err := afero.ReadDir(fsys, utils.MigrationsDir)
require.NoError(t, err)
require.Len(t, files, 1)
contents, err := afero.ReadFile(fsys, filepath.Join(utils.MigrationsDir, files[0].Name()))
require.NoError(t, err)
assert.Equal(t, []byte(generated), contents)
})

t.Run("throws error on failure to diff target", func(t *testing.T) {
// Setup in-memory fs
fsys := afero.NewMemMapFs()
Expand Down
2 changes: 1 addition & 1 deletion apps/cli-go/internal/db/diff/shadow.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func PrepareShadowSource(ctx context.Context, schema []string, targetLocal bool,
if len(declared) > 0 {
override := shadowConfig
override.Database = "contrib_regression"
if usePgDelta {
if shouldApplyDeclarativeWithPgDelta(usePgDelta) {
declDir := utils.GetDeclarativeDir()
if exists, _ := afero.DirExists(fsys, declDir); exists {
if err := pgdelta.ApplyDeclarative(ctx, override, fsys); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion apps/cli-go/internal/migration/apply/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func applySeedFiles(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {
}

func applySchemaFiles(ctx context.Context, conn *pgx.Conn, fsys fs.FS) error {
declared, err := utils.Config.Db.Migrations.SchemaPaths.Files(fsys)
declared, err := utils.Config.Db.Migrations.SchemaPaths.SQLFiles(fsys)
if len(declared) == 0 {
return err
}
Expand Down
51 changes: 51 additions & 0 deletions apps/cli-go/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,19 @@ type Glob []string
// Match the glob patterns in the given FS to get a deduplicated
// array of all migrations files to apply in the declared order.
func (g Glob) Files(fsys fs.FS) ([]string, error) {
return g.files(fsys, nil)
}

// SQLFiles matches glob patterns and expands directory matches recursively to
// SQL files. Pattern order is preserved, and directory contents are sorted for
// deterministic application.
func (g Glob) SQLFiles(fsys fs.FS) ([]string, error) {
return g.files(fsys, func(path string, entry fs.DirEntry) bool {
return entry.Type().IsRegular() && filepath.Ext(path) == ".sql"
})
}

func (g Glob) files(fsys fs.FS, expandDir func(string, fs.DirEntry) bool) ([]string, error) {
var result []string
var allErrors []error
set := make(map[string]struct{})
Expand All @@ -115,6 +128,27 @@ func (g Glob) Files(fsys fs.FS) ([]string, error) {
// Remove duplicates
for _, item := range matches {
fp := filepath.ToSlash(item)
if expandDir != nil {
info, err := fs.Stat(fsys, fp)
if err != nil {
allErrors = append(allErrors, errors.Errorf("failed to stat matched file: %w", err))
continue
}
if info.IsDir() {
files, err := walkMatchedDir(fsys, fp, expandDir)
if err != nil {
allErrors = append(allErrors, err)
continue
}
for _, file := range files {
if _, exists := set[file]; !exists {
set[file] = struct{}{}
result = append(result, file)
}
}
continue
}
}
if _, exists := set[fp]; !exists {
set[fp] = struct{}{}
result = append(result, fp)
Expand All @@ -124,6 +158,23 @@ func (g Glob) Files(fsys fs.FS) ([]string, error) {
return result, errors.Join(allErrors...)
}

func walkMatchedDir(fsys fs.FS, dir string, include func(string, fs.DirEntry) bool) ([]string, error) {
var files []string
if err := fs.WalkDir(fsys, dir, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if include(path, entry) {
files = append(files, filepath.ToSlash(path))
}
return nil
}); err != nil {
return nil, errors.Errorf("failed to walk matched directory: %w", err)
}
sort.Strings(files)
return files, nil
}

// We follow these rules when adding new config:
// 1. Update init_config.toml (and init_config.test.toml) with the new key, default value, and comments to explain usage.
// 2. Update config struct with new field and toml tag (spelled in snake_case).
Expand Down
44 changes: 44 additions & 0 deletions apps/cli-go/pkg/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,50 @@ func TestGlobFiles(t *testing.T) {
})
}

func TestGlobSQLFiles(t *testing.T) {
t.Run("expands directory entries in declared order", func(t *testing.T) {
fsys := fs.MapFS{
"supabase/schemas/z_function.sql": &fs.MapFile{Data: []byte("select 1;")},
"supabase/schemas/tables/a_table.sql": &fs.MapFile{Data: []byte("select 2;")},
"supabase/schemas/tables/nested/b_table.sql": &fs.MapFile{Data: []byte("select 3;")},
"supabase/schemas/tables/readme.md": &fs.MapFile{Data: []byte("ignored")},
}
g := Glob{
"supabase/schemas/z_function.sql",
"supabase/schemas/tables",
}

files, err := g.SQLFiles(fsys)

assert.NoError(t, err)
assert.Equal(t, []string{
"supabase/schemas/z_function.sql",
"supabase/schemas/tables/a_table.sql",
"supabase/schemas/tables/nested/b_table.sql",
}, files)
})

t.Run("deduplicates explicit files and directory matches", func(t *testing.T) {
fsys := fs.MapFS{
"supabase/database/a.sql": &fs.MapFile{Data: []byte("select 1;")},
"supabase/database/b.sql": &fs.MapFile{Data: []byte("select 2;")},
}
g := Glob{
"supabase/database/a.sql",
"supabase/database",
"supabase/database/*.sql",
}

files, err := g.SQLFiles(fsys)

assert.NoError(t, err)
assert.Equal(t, []string{
"supabase/database/a.sql",
"supabase/database/b.sql",
}, files)
})
}

func TestLoadFunctionImportMap(t *testing.T) {
t.Run("uses deno.json as import map when present", func(t *testing.T) {
config := NewConfig()
Expand Down
4 changes: 2 additions & 2 deletions apps/cli-go/pkg/config/templates/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ max_client_conn = 100
[db.migrations]
# If disabled, migrations will be skipped during a db push or reset.
enabled = true
# Specifies an ordered list of schema files that describe your database.
# Supports glob patterns relative to supabase directory: "./schemas/*.sql"
# Specifies an ordered list of schema files, directories, or glob patterns that describe your database.
# Supports paths relative to supabase directory: "./schemas/*.sql", "./database".
schema_paths = []

[db.seed]
Expand Down
2 changes: 1 addition & 1 deletion apps/cli-go/pkg/migration/seed.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func getRemoteSeeds(ctx context.Context, conn *pgx.Conn) (map[string]string, err
}

func GetPendingSeeds(ctx context.Context, locals config.Glob, conn *pgx.Conn, fsys fs.FS) ([]SeedFile, error) {
locals, err := locals.Files(fsys)
locals, err := locals.SQLFiles(fsys)
if err != nil {
fmt.Fprintln(os.Stderr, "WARN:", err)
}
Expand Down
4 changes: 2 additions & 2 deletions apps/cli/src/shared/init/project-init.templates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ max_client_conn = 100
[db.migrations]
# If disabled, migrations will be skipped during a db push or reset.
enabled = true
# Specifies an ordered list of schema files that describe your database.
# Supports glob patterns relative to supabase directory: "./schemas/*.sql"
# Specifies an ordered list of schema files, directories, or glob patterns that describe your database.
# Supports paths relative to supabase directory: "./schemas/*.sql", "./database".
schema_paths = []

[db.seed]
Expand Down
Loading