diff --git a/apps/cli-go/internal/db/diff/diff.go b/apps/cli-go/internal/db/diff/diff.go index 5d2790e5b9..85c9c09c9c 100644 --- a/apps/cli-go/internal/db/diff/diff.go +++ b/apps/cli-go/internal/db/diff/diff.go @@ -50,6 +50,13 @@ func Run(ctx context.Context, schema []string, file string, config pgconn.Config } func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) { + if schemas := utils.Config.Db.Migrations.SchemaPaths; len(schemas) > 0 { + return schemas.SQLFiles( + afero.NewIOFS(fsys), + configpkg.WithSkipEmptyGlobs(), + configpkg.WithErrorOnAllSkippedGlobs(), + ) + } // When pg-delta is enabled, declarative path is the source of truth (config or default). if utils.IsPgDeltaEnabled() { declDir := utils.GetDeclarativeDir() @@ -70,13 +77,6 @@ func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) { return declared, nil } } - if schemas := utils.Config.Db.Migrations.SchemaPaths; len(schemas) > 0 { - return schemas.Files( - afero.NewIOFS(fsys), - configpkg.WithSkipEmptyGlobs(), - configpkg.WithErrorOnAllSkippedGlobs(), - ) - } if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil { return nil, errors.Errorf("failed to check schemas: %w", err) } else if !exists { @@ -100,6 +100,24 @@ func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) { return declared, nil } +func shouldApplyDeclarativeWithPgDelta(usePgDelta bool) bool { + if !usePgDelta { + return false + } + schemas := utils.Config.Db.Migrations.SchemaPaths + if len(schemas) == 0 { + return true + } + if len(schemas) != 1 { + return false + } + return cleanSchemaPath(schemas[0]) == cleanSchemaPath(utils.GetDeclarativeDir()) +} + +func cleanSchemaPath(path string) string { + return filepath.ToSlash(filepath.Clean(path)) +} + // https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6 var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`) diff --git a/apps/cli-go/internal/db/diff/diff_test.go b/apps/cli-go/internal/db/diff/diff_test.go index 2a6a2d4ca4..9f60d817bf 100644 --- a/apps/cli-go/internal/db/diff/diff_test.go +++ b/apps/cli-go/internal/db/diff/diff_test.go @@ -24,6 +24,7 @@ import ( "github.com/supabase/cli/internal/testing/helper" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" + pkgconfig "github.com/supabase/cli/pkg/config" "github.com/supabase/cli/pkg/migration" "github.com/supabase/cli/pkg/pgtest" ) @@ -36,6 +37,80 @@ var dbConfig = pgconn.Config{ Database: "postgres", } +func TestLoadDeclaredSchemas(t *testing.T) { + t.Run("respects schema_paths order when pg-delta declarative dir exists", func(t *testing.T) { + originalConfig := utils.Config + t.Cleanup(func() { utils.Config = originalConfig }) + utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{ + "supabase/schemas/z_function.sql", + "supabase/schemas/a_table.sql", + } + utils.Config.Experimental.PgDelta = &pkgconfig.PgDeltaConfig{ + Enabled: true, + DeclarativeSchemaPath: utils.SchemasDir, + } + fsys := afero.NewMemMapFs() + require.NoError(t, fsys.MkdirAll(utils.SchemasDir, 0755)) + require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/a_table.sql", []byte("create table a();"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/z_function.sql", []byte("create function z() returns void language sql as $$ select 1 $$;"), 0644)) + + declared, err := loadDeclaredSchemas(fsys) + + require.NoError(t, err) + assert.Equal(t, []string{ + "supabase/schemas/z_function.sql", + "supabase/schemas/a_table.sql", + }, declared) + }) + + t.Run("expands schema_paths directory entries deterministically", func(t *testing.T) { + originalConfig := utils.Config + t.Cleanup(func() { utils.Config = originalConfig }) + utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{utils.DeclarativeDir} + fsys := afero.NewMemMapFs() + require.NoError(t, fsys.MkdirAll(filepath.Join(utils.DeclarativeDir, "nested"), 0755)) + require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.DeclarativeDir, "nested", "b.sql"), []byte("select 2;"), 0644)) + require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.DeclarativeDir, "a.sql"), []byte("select 1;"), 0644)) + + declared, err := loadDeclaredSchemas(fsys) + + require.NoError(t, err) + assert.Equal(t, []string{ + filepath.Join(utils.DeclarativeDir, "a.sql"), + filepath.Join(utils.DeclarativeDir, "nested", "b.sql"), + }, declared) + }) +} + +func TestShouldApplyDeclarativeWithPgDelta(t *testing.T) { + t.Run("uses pg-delta declarative apply when no schema_paths override is configured", func(t *testing.T) { + originalConfig := utils.Config + t.Cleanup(func() { utils.Config = originalConfig }) + utils.Config.Db.Migrations.SchemaPaths = nil + + assert.True(t, shouldApplyDeclarativeWithPgDelta(true)) + }) + + t.Run("uses pg-delta declarative apply when schema_paths points at the declarative dir", func(t *testing.T) { + originalConfig := utils.Config + t.Cleanup(func() { utils.Config = originalConfig }) + utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{utils.DeclarativeDir + "/"} + + assert.True(t, shouldApplyDeclarativeWithPgDelta(true)) + }) + + t.Run("uses ordered migration apply for explicit schema_paths files", func(t *testing.T) { + originalConfig := utils.Config + t.Cleanup(func() { utils.Config = originalConfig }) + utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{ + "supabase/schemas/z_function.sql", + "supabase/schemas/a_table.sql", + } + + assert.False(t, shouldApplyDeclarativeWithPgDelta(true)) + }) +} + func TestRun(t *testing.T) { t.Run("runs migra diff", func(t *testing.T) { // Setup in-memory fs @@ -98,6 +173,91 @@ func TestRun(t *testing.T) { assert.Equal(t, []byte(diff), contents) }) + t.Run("applies schema_paths in order before saving generated diff", func(t *testing.T) { + originalConfig := utils.Config + t.Cleanup(func() { utils.Config = originalConfig }) + utils.Config.Db.MajorVersion = 14 + utils.Config.Db.ShadowPort = 54320 + utils.Config.Db.Migrations.SchemaPaths = pkgconfig.Glob{ + "supabase/schemas/z_function.sql", + "supabase/schemas/a_table.sql", + } + utils.Config.Experimental.PgDelta = &pkgconfig.PgDeltaConfig{ + Enabled: true, + DeclarativeSchemaPath: utils.SchemasDir, + } + utils.GlobalsSql = "create schema public" + utils.InitialSchemaPg14Sql = "create schema private" + functionSQL := "create function public.z_function() returns integer language sql as $$ select 1 $$" + tableSQL := "create table public.a_table (id integer default public.z_function())" + generated := functionSQL + ";\n" + tableSQL + ";\n" + fsys := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/a_table.sql", []byte(tableSQL), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/schemas/z_function.sql", []byte(functionSQL), 0644)) + require.NoError(t, apitest.MockDocker(utils.Docker)) + defer gock.OffAll() + apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.Config.Db.Image), "test-shadow-db") + gock.New(utils.Docker.DaemonHost()). + Get("/v" + utils.Docker.ClientVersion() + "/containers/test-shadow-db/json"). + Reply(http.StatusOK). + JSON(container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ + Running: true, + Health: &container.Health{Status: types.Healthy}, + }, + }}) + gock.New(utils.Docker.DaemonHost()). + Delete("/v" + utils.Docker.ClientVersion() + "/containers/test-shadow-db"). + Reply(http.StatusOK) + shadowConn := pgtest.NewConn() + defer shadowConn.Close(t) + shadowConn.Query(utils.GlobalsSql). + Reply("CREATE SCHEMA"). + Query(utils.InitialSchemaPg14Sql). + Reply("CREATE SCHEMA") + helper.MockApiPrivilegesRevoke(shadowConn). + Query(CREATE_TEMPLATE). + Reply("CREATE DATABASE") + declaredConn := pgtest.NewConn() + defer declaredConn.Close(t) + declaredConn.Query(functionSQL). + Reply("CREATE FUNCTION"). + Query(tableSQL). + Reply("CREATE TABLE") + diffCalled := false + differ := func(_ context.Context, _, target pgconn.Config, schema []string, _ ...func(*pgx.ConnConfig)) (string, error) { + diffCalled = true + assert.Equal(t, "contrib_regression", target.Database) + assert.Equal(t, []string{"public"}, schema) + return generated, nil + } + localConfig := pgconn.Config{ + Host: utils.Config.Hostname, + Port: utils.Config.Db.Port, + User: "postgres", + Password: utils.Config.Db.Password, + Database: "postgres", + } + + err := Run(context.Background(), []string{"public"}, "ordered_schema", localConfig, differ, true, fsys, func(cc *pgx.ConnConfig) { + if cc.Database == "contrib_regression" { + declaredConn.Intercept(cc) + } else { + shadowConn.Intercept(cc) + } + }) + + require.NoError(t, err) + assert.True(t, diffCalled) + assert.Empty(t, apitest.ListUnmatchedRequests()) + files, err := afero.ReadDir(fsys, utils.MigrationsDir) + require.NoError(t, err) + require.Len(t, files, 1) + contents, err := afero.ReadFile(fsys, filepath.Join(utils.MigrationsDir, files[0].Name())) + require.NoError(t, err) + assert.Equal(t, []byte(generated), contents) + }) + t.Run("throws error on failure to diff target", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() diff --git a/apps/cli-go/internal/db/diff/shadow.go b/apps/cli-go/internal/db/diff/shadow.go index 8012c565e8..2ebd13591f 100644 --- a/apps/cli-go/internal/db/diff/shadow.go +++ b/apps/cli-go/internal/db/diff/shadow.go @@ -67,7 +67,7 @@ func PrepareShadowSource(ctx context.Context, schema []string, targetLocal bool, if len(declared) > 0 { override := shadowConfig override.Database = "contrib_regression" - if usePgDelta { + if shouldApplyDeclarativeWithPgDelta(usePgDelta) { declDir := utils.GetDeclarativeDir() if exists, _ := afero.DirExists(fsys, declDir); exists { if err := pgdelta.ApplyDeclarative(ctx, override, fsys); err != nil { diff --git a/apps/cli-go/internal/migration/apply/apply.go b/apps/cli-go/internal/migration/apply/apply.go index 16fdd63611..18bd330ea6 100644 --- a/apps/cli-go/internal/migration/apply/apply.go +++ b/apps/cli-go/internal/migration/apply/apply.go @@ -49,7 +49,7 @@ func applySeedFiles(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error { } func applySchemaFiles(ctx context.Context, conn *pgx.Conn, fsys fs.FS) error { - declared, err := utils.Config.Db.Migrations.SchemaPaths.Files(fsys) + declared, err := utils.Config.Db.Migrations.SchemaPaths.SQLFiles(fsys) if len(declared) == 0 { return err } diff --git a/apps/cli-go/pkg/config/config.go b/apps/cli-go/pkg/config/config.go index b9f253230a..b55a52b27e 100644 --- a/apps/cli-go/pkg/config/config.go +++ b/apps/cli-go/pkg/config/config.go @@ -116,9 +116,22 @@ func WithErrorOnAllSkippedGlobs() GlobOption { } } +func (g Glob) Files(fsys fs.FS, options ...GlobOption) ([]string, error) { + return g.files(fsys, nil, options...) +} + +// SQLFiles matches glob patterns and expands directory matches recursively to +// SQL files. Pattern order is preserved, and directory contents are sorted for +// deterministic application. +func (g Glob) SQLFiles(fsys fs.FS, options ...GlobOption) ([]string, error) { + return g.files(fsys, func(path string, entry fs.DirEntry) bool { + return entry.Type().IsRegular() && filepath.Ext(path) == ".sql" + }, options...) +} + // Match the glob patterns in the given FS to get a deduplicated // array of all migrations files to apply in the declared order. -func (g Glob) Files(fsys fs.FS, options ...GlobOption) ([]string, error) { +func (g Glob) files(fsys fs.FS, expandDir func(string, fs.DirEntry) bool, options ...GlobOption) ([]string, error) { opts := globOptions{} for _, apply := range options { apply(&opts) @@ -143,6 +156,27 @@ func (g Glob) Files(fsys fs.FS, options ...GlobOption) ([]string, error) { // Remove duplicates for _, item := range matches { fp := filepath.ToSlash(item) + if expandDir != nil { + info, err := fs.Stat(fsys, fp) + if err != nil { + allErrors = append(allErrors, errors.Errorf("failed to stat matched file: %w", err)) + continue + } + if info.IsDir() { + files, err := walkMatchedDir(fsys, fp, expandDir) + if err != nil { + allErrors = append(allErrors, err) + continue + } + for _, file := range files { + if _, exists := set[file]; !exists { + set[file] = struct{}{} + result = append(result, file) + } + } + continue + } + } if _, exists := set[fp]; !exists { set[fp] = struct{}{} result = append(result, fp) @@ -157,6 +191,23 @@ func (g Glob) Files(fsys fs.FS, options ...GlobOption) ([]string, error) { return result, errors.Join(allErrors...) } +func walkMatchedDir(fsys fs.FS, dir string, include func(string, fs.DirEntry) bool) ([]string, error) { + var files []string + if err := fs.WalkDir(fsys, dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if include(path, entry) { + files = append(files, filepath.ToSlash(path)) + } + return nil + }); err != nil { + return nil, errors.Errorf("failed to walk matched directory: %w", err) + } + sort.Strings(files) + return files, nil +} + func hasGlobMeta(pattern string) bool { return strings.ContainsAny(pattern, `*?[`) } diff --git a/apps/cli-go/pkg/config/config_test.go b/apps/cli-go/pkg/config/config_test.go index 6c2697ca0e..aa0570ec34 100644 --- a/apps/cli-go/pkg/config/config_test.go +++ b/apps/cli-go/pkg/config/config_test.go @@ -706,6 +706,50 @@ func TestGlobFiles(t *testing.T) { }) } +func TestGlobSQLFiles(t *testing.T) { + t.Run("expands directory entries in declared order", func(t *testing.T) { + fsys := fs.MapFS{ + "supabase/schemas/z_function.sql": &fs.MapFile{Data: []byte("select 1;")}, + "supabase/schemas/tables/a_table.sql": &fs.MapFile{Data: []byte("select 2;")}, + "supabase/schemas/tables/nested/b_table.sql": &fs.MapFile{Data: []byte("select 3;")}, + "supabase/schemas/tables/readme.md": &fs.MapFile{Data: []byte("ignored")}, + } + g := Glob{ + "supabase/schemas/z_function.sql", + "supabase/schemas/tables", + } + + files, err := g.SQLFiles(fsys) + + assert.NoError(t, err) + assert.Equal(t, []string{ + "supabase/schemas/z_function.sql", + "supabase/schemas/tables/a_table.sql", + "supabase/schemas/tables/nested/b_table.sql", + }, files) + }) + + t.Run("deduplicates explicit files and directory matches", func(t *testing.T) { + fsys := fs.MapFS{ + "supabase/database/a.sql": &fs.MapFile{Data: []byte("select 1;")}, + "supabase/database/b.sql": &fs.MapFile{Data: []byte("select 2;")}, + } + g := Glob{ + "supabase/database/a.sql", + "supabase/database", + "supabase/database/*.sql", + } + + files, err := g.SQLFiles(fsys) + + assert.NoError(t, err) + assert.Equal(t, []string{ + "supabase/database/a.sql", + "supabase/database/b.sql", + }, files) + }) +} + func TestLoadFunctionImportMap(t *testing.T) { t.Run("uses deno.json as import map when present", func(t *testing.T) { config := NewConfig() diff --git a/apps/cli-go/pkg/config/templates/config.toml b/apps/cli-go/pkg/config/templates/config.toml index 56cc27beac..98e034f8d8 100644 --- a/apps/cli-go/pkg/config/templates/config.toml +++ b/apps/cli-go/pkg/config/templates/config.toml @@ -59,8 +59,8 @@ max_client_conn = 100 [db.migrations] # If disabled, migrations will be skipped during a db push or reset. enabled = true -# Specifies an ordered list of schema files that describe your database. -# Supports glob patterns relative to supabase directory: "./schemas/*.sql" +# Specifies an ordered list of schema files, directories, or glob patterns that describe your database. +# Supports paths relative to supabase directory: "./schemas/*.sql", "./database". schema_paths = [] [db.seed] diff --git a/apps/cli-go/pkg/migration/seed.go b/apps/cli-go/pkg/migration/seed.go index 89a049878a..2e8fff71c8 100644 --- a/apps/cli-go/pkg/migration/seed.go +++ b/apps/cli-go/pkg/migration/seed.go @@ -32,7 +32,7 @@ func getRemoteSeeds(ctx context.Context, conn *pgx.Conn) (map[string]string, err } func GetPendingSeeds(ctx context.Context, locals config.Glob, conn *pgx.Conn, fsys fs.FS) ([]SeedFile, error) { - locals, err := locals.Files(fsys) + locals, err := locals.SQLFiles(fsys) if err != nil { fmt.Fprintln(os.Stderr, "WARN:", err) } diff --git a/apps/cli/src/shared/init/project-init.templates.ts b/apps/cli/src/shared/init/project-init.templates.ts index 6415e44a22..f3a41b6db8 100644 --- a/apps/cli/src/shared/init/project-init.templates.ts +++ b/apps/cli/src/shared/init/project-init.templates.ts @@ -59,8 +59,8 @@ max_client_conn = 100 [db.migrations] # If disabled, migrations will be skipped during a db push or reset. enabled = true -# Specifies an ordered list of schema files that describe your database. -# Supports glob patterns relative to supabase directory: "./schemas/*.sql" +# Specifies an ordered list of schema files, directories, or glob patterns that describe your database. +# Supports paths relative to supabase directory: "./schemas/*.sql", "./database". schema_paths = [] [db.seed] diff --git a/apps/docs/public/cli/config.schema.json b/apps/docs/public/cli/config.schema.json index 28062d0891..0503269e77 100644 --- a/apps/docs/public/cli/config.schema.json +++ b/apps/docs/public/cli/config.schema.json @@ -2420,9 +2420,9 @@ "type": "array", "items": { "type": "string", - "description": "Schema file path or glob relative to the supabase directory." + "description": "Schema file path, directory, or glob relative to the supabase directory." }, - "description": "Ordered list of schema files that describe your database.", + "description": "Ordered list of schema files, directories, or glob patterns that describe your database.", "default": [] } }, @@ -5965,9 +5965,9 @@ "type": "array", "items": { "type": "string", - "description": "Schema file path or glob relative to the supabase directory." + "description": "Schema file path, directory, or glob relative to the supabase directory." }, - "description": "Ordered list of schema files that describe your database.", + "description": "Ordered list of schema files, directories, or glob patterns that describe your database.", "default": [] } }, diff --git a/packages/config/src/db.ts b/packages/config/src/db.ts index 1bc6c57c9f..28e1fdabe6 100644 --- a/packages/config/src/db.ts +++ b/packages/config/src/db.ts @@ -132,13 +132,14 @@ export const db = Schema.Struct({ }).pipe(Schema.withDecodingDefaultKey(Effect.succeed(defaultMigrationsEnabled))), schema_paths: Schema.Array( Schema.String.annotate({ - description: "Schema file path or glob relative to the supabase directory.", + description: "Schema file path, directory, or glob relative to the supabase directory.", tags, }), ) .annotate({ default: defaultSchemaPaths, - description: "Ordered list of schema files that describe your database.", + description: + "Ordered list of schema files, directories, or glob patterns that describe your database.", tags, }) .pipe(Schema.withDecodingDefaultKey(Effect.succeed([...defaultSchemaPaths]))),