From 658344488d498c03e0641567072219c4723c6680 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Sun, 10 May 2026 11:35:54 +0300 Subject: [PATCH 1/2] Drop old migrations and adopt gomigrate This change drops old ad-hoc database migration code. There were some migrations that we were carrying on from really old versions of GARM. This change drops all those migrations and adopts gomigrate moving forward. Upgrades will have to first go to version v0.2.1 and then move to the v0.2.2. Signed-off-by: Gabriel Adrian Samfira --- config/config.go | 2 +- database/sql/migrations/0001_baseline.go | 29 + .../migrations/0001_baseline_file_objects.go | 29 + database/sql/migrations/migrations.go | 56 ++ database/sql/sql.go | 594 +----------------- go.mod | 1 + go.sum | 2 + .../go-gormigrate/gormigrate/v2/.gitignore | 5 + .../go-gormigrate/gormigrate/v2/CHANGELOG.md | 113 ++++ .../go-gormigrate/gormigrate/v2/LICENSE | 8 + .../go-gormigrate/gormigrate/v2/README.md | 232 +++++++ .../go-gormigrate/gormigrate/v2/Taskfile.yml | 85 +++ .../go-gormigrate/gormigrate/v2/gormigrate.go | 493 +++++++++++++++ vendor/modules.txt | 3 + 14 files changed, 1088 insertions(+), 564 deletions(-) create mode 100644 database/sql/migrations/0001_baseline.go create mode 100644 database/sql/migrations/0001_baseline_file_objects.go create mode 100644 database/sql/migrations/migrations.go create mode 100644 vendor/github.com/go-gormigrate/gormigrate/v2/.gitignore create mode 100644 vendor/github.com/go-gormigrate/gormigrate/v2/CHANGELOG.md create mode 100644 vendor/github.com/go-gormigrate/gormigrate/v2/LICENSE create mode 100644 vendor/github.com/go-gormigrate/gormigrate/v2/README.md create mode 100644 vendor/github.com/go-gormigrate/gormigrate/v2/Taskfile.yml create mode 100644 vendor/github.com/go-gormigrate/gormigrate/v2/gormigrate.go diff --git a/config/config.go b/config/config.go index f960cb9c..7a90371b 100644 --- a/config/config.go +++ b/config/config.go @@ -601,7 +601,7 @@ func (s *SQLite) BlobDBFile() (string, error) { } func (s *SQLite) connectionStringForDBFile(dbFile string) string { - connectionString := fmt.Sprintf("%s?_journal_mode=WAL&_foreign_keys=ON&_txlock=immediate", dbFile) + connectionString := fmt.Sprintf("%s?_journal_mode=WAL&_foreign_keys=ON&_txlock=immediate&_auto_vacuum=incremental", dbFile) if s.BusyTimeoutSeconds > 0 { timeout := s.BusyTimeoutSeconds * 1000 connectionString = fmt.Sprintf("%s&_busy_timeout=%d", connectionString, timeout) diff --git a/database/sql/migrations/0001_baseline.go b/database/sql/migrations/0001_baseline.go new file mode 100644 index 00000000..285e9aa1 --- /dev/null +++ b/database/sql/migrations/0001_baseline.go @@ -0,0 +1,29 @@ +// Copyright 2026 Cloudbase Solutions SRL +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package migrations + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +func init() { + Register(&gormigrate.Migration{ + ID: "0001_baseline", + Migrate: func(tx *gorm.DB) error { + return nil + }, + }) +} diff --git a/database/sql/migrations/0001_baseline_file_objects.go b/database/sql/migrations/0001_baseline_file_objects.go new file mode 100644 index 00000000..bea3d367 --- /dev/null +++ b/database/sql/migrations/0001_baseline_file_objects.go @@ -0,0 +1,29 @@ +// Copyright 2026 Cloudbase Solutions SRL +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package migrations + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +func init() { + RegisterFileObjects(&gormigrate.Migration{ + ID: "0001_baseline", + Migrate: func(tx *gorm.DB) error { + return nil + }, + }) +} diff --git a/database/sql/migrations/migrations.go b/database/sql/migrations/migrations.go new file mode 100644 index 00000000..0509a4a7 --- /dev/null +++ b/database/sql/migrations/migrations.go @@ -0,0 +1,56 @@ +// Copyright 2026 Cloudbase Solutions SRL +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package migrations + +import ( + "sort" + + "github.com/go-gormigrate/gormigrate/v2" +) + +var ( + registry []*gormigrate.Migration + fileObjectsRegistry []*gormigrate.Migration +) + +// Register adds a migration to the main database registry. Each migration file +// should call this in an init() function. +func Register(m *gormigrate.Migration) { + registry = append(registry, m) +} + +// RegisterFileObjects adds a migration to the file objects database registry. +func RegisterFileObjects(m *gormigrate.Migration) { + fileObjectsRegistry = append(fileObjectsRegistry, m) +} + +func sorted(migrations []*gormigrate.Migration) []*gormigrate.Migration { + result := make([]*gormigrate.Migration, len(migrations)) + copy(result, migrations) + sort.Slice(result, func(i, j int) bool { + return result[i].ID < result[j].ID + }) + return result +} + +// All returns all registered main database migrations sorted alphanumerically by ID. +func All() []*gormigrate.Migration { + return sorted(registry) +} + +// AllFileObjects returns all registered file objects migrations sorted alphanumerically by ID. +func AllFileObjects() []*gormigrate.Migration { + return sorted(fileObjectsRegistry) +} diff --git a/database/sql/sql.go b/database/sql/sql.go index 355f6653..bcb19e9a 100644 --- a/database/sql/sql.go +++ b/database/sql/sql.go @@ -20,9 +20,6 @@ import ( "errors" "fmt" "log/slog" - "net/url" - "regexp" - "strings" "time" "gorm.io/driver/mysql" @@ -30,6 +27,10 @@ import ( "gorm.io/gorm" "gorm.io/gorm/logger" + "github.com/go-gormigrate/gormigrate/v2" + + "github.com/cloudbase/garm/database/sql/migrations" + runnerErrors "github.com/cloudbase/garm-provider-common/errors" commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm/auth" @@ -175,115 +176,6 @@ func (s *sqlDatabase) runSQLiteMaintenance(conn *gorm.DB, dbName string) { } } -var renameTemplate = ` -PRAGMA foreign_keys = OFF; -BEGIN TRANSACTION; - -ALTER TABLE %s RENAME TO %s_old; -COMMIT; -` - -var restoreNameTemplate = ` -PRAGMA foreign_keys = OFF; -BEGIN TRANSACTION; -DROP TABLE IF EXISTS %s; -ALTER TABLE %s_old RENAME TO %s; -COMMIT; -` - -var copyContentsTemplate = ` -PRAGMA foreign_keys = OFF; -BEGIN TRANSACTION; -INSERT INTO %s SELECT * FROM %s_old; -DROP TABLE %s_old; - -COMMIT; -` - -func (s *sqlDatabase) cascadeMigrationSQLite(model interface{}, name string, justDrop bool) error { - if !s.conn.Migrator().HasTable(name) { - return nil - } - defer s.conn.Exec("PRAGMA foreign_keys = ON;") - - var data string - var indexes []string - if err := s.conn.Raw(fmt.Sprintf("select sql from sqlite_master where type='table' and tbl_name='%s'", name)).Scan(&data).Error; err != nil { - if !errors.Is(err, gorm.ErrRecordNotFound) { - return fmt.Errorf("failed to get table %s: %w", name, err) - } - } - - if err := s.conn.Raw(fmt.Sprintf("SELECT name FROM sqlite_master WHERE type == 'index' AND tbl_name == '%s' and name not like 'sqlite_%%'", name)).Scan(&indexes).Error; err != nil { - if !errors.Is(err, gorm.ErrRecordNotFound) { - return fmt.Errorf("failed to get table indexes %s: %w", name, err) - } - } - - if strings.Contains(data, "ON DELETE") { - return nil - } - - if justDrop { - if err := s.conn.Migrator().DropTable(model); err != nil { - return fmt.Errorf("failed to drop table %s: %w", name, err) - } - return nil - } - - for _, index := range indexes { - if err := s.conn.Migrator().DropIndex(model, index); err != nil { - return fmt.Errorf("failed to drop index %s: %w", index, err) - } - } - - err := s.conn.Exec(fmt.Sprintf(renameTemplate, name, name)).Error - if err != nil { - return fmt.Errorf("failed to rename table %s: %w", name, err) - } - - if model != nil { - if err := s.conn.Migrator().AutoMigrate(model); err != nil { - if err := s.conn.Exec(fmt.Sprintf(restoreNameTemplate, name, name, name)).Error; err != nil { - slog.With(slog.Any("error", err)).Error("failed to restore table", "table", name) - } - return fmt.Errorf("failed to create table %s: %w", name, err) - } - } - err = s.conn.Exec(fmt.Sprintf(copyContentsTemplate, name, name, name)).Error - if err != nil { - return fmt.Errorf("failed to copy contents to table %s: %w", name, err) - } - - return nil -} - -func (s *sqlDatabase) cascadeMigration() error { - switch s.cfg.DbBackend { - case config.SQLiteBackend: - if err := s.cascadeMigrationSQLite(&Address{}, "addresses", true); err != nil { - return fmt.Errorf("failed to drop table addresses: %w", err) - } - - if err := s.cascadeMigrationSQLite(&InstanceStatusUpdate{}, "instance_status_updates", true); err != nil { - return fmt.Errorf("failed to drop table instance_status_updates: %w", err) - } - - if err := s.cascadeMigrationSQLite(&Tag{}, "pool_tags", false); err != nil { - return fmt.Errorf("failed to migrate addresses: %w", err) - } - - if err := s.cascadeMigrationSQLite(&WorkflowJob{}, "workflow_jobs", false); err != nil { - return fmt.Errorf("failed to migrate addresses: %w", err) - } - case config.MySQLBackend: - return nil - default: - return fmt.Errorf("invalid db backend: %s", s.cfg.DbBackend) - } - return nil -} - func (s *sqlDatabase) ensureGithubEndpoint() error { // Create the default Github endpoint. createEndpointParams := params.CreateGithubEndpointParams{ @@ -312,220 +204,18 @@ func (s *sqlDatabase) ensureGithubEndpoint() error { return nil } -func (s *sqlDatabase) migrateCredentialsToDB() (err error) { - s.conn.Exec("PRAGMA foreign_keys = OFF") - defer s.conn.Exec("PRAGMA foreign_keys = ON") - - adminUser, err := s.GetAdminUser(s.ctx) - if err != nil { - if errors.Is(err, runnerErrors.ErrNotFound) { - // Admin user doesn't exist. This is a new deploy. Nothing to migrate. - return nil - } - return fmt.Errorf("error getting admin user: %w", err) - } - - // Impersonate the admin user. We're migrating from config credentials to - // database credentials. At this point, there is no other user than the admin - // user. GARM is not yet multi-user, so it's safe to assume we only have this - // one user. - adminCtx := context.Background() - adminCtx = auth.PopulateContext(adminCtx, adminUser, nil) - - slog.Info("migrating credentials to DB") - slog.Info("creating github endpoints table") - if err := s.conn.AutoMigrate(&GithubEndpoint{}); err != nil { - return fmt.Errorf("error migrating github endpoints: %w", err) - } - - defer func() { - if err != nil { - slog.With(slog.Any("error", err)).Error("rolling back github github endpoints table") - s.conn.Migrator().DropTable(&GithubEndpoint{}) - } - }() - - slog.Info("creating github credentials table") - if err := s.conn.AutoMigrate(&GithubCredentials{}); err != nil { - return fmt.Errorf("error migrating github credentials: %w", err) - } - - defer func() { - if err != nil { - slog.With(slog.Any("error", err)).Error("rolling back github github credentials table") - s.conn.Migrator().DropTable(&GithubCredentials{}) - } - }() - - // Nothing to migrate. - if len(s.cfg.MigrateCredentials) == 0 { - return nil - } - - slog.Info("importing credentials from config") - for _, cred := range s.cfg.MigrateCredentials { - slog.Info("importing credential", "name", cred.Name) - parsed, err := url.Parse(cred.BaseEndpoint()) - if err != nil { - return fmt.Errorf("error parsing base URL: %w", err) - } - - certBundle, err := cred.CACertBundle() - if err != nil { - return fmt.Errorf("error getting CA cert bundle: %w", err) - } - hostname := parsed.Hostname() - createParams := params.CreateGithubEndpointParams{ - Name: hostname, - Description: fmt.Sprintf("Endpoint for %s", hostname), - APIBaseURL: cred.APIEndpoint(), - BaseURL: cred.BaseEndpoint(), - UploadBaseURL: cred.UploadEndpoint(), - CACertBundle: certBundle, - } - - var endpoint params.ForgeEndpoint - endpoint, err = s.GetGithubEndpoint(adminCtx, hostname) - if err != nil { - if !errors.Is(err, runnerErrors.ErrNotFound) { - return fmt.Errorf("error getting github endpoint: %w", err) - } - endpoint, err = s.CreateGithubEndpoint(adminCtx, createParams) - if err != nil { - return fmt.Errorf("error creating default github endpoint: %w", err) - } - } - - credParams := params.CreateGithubCredentialsParams{ - Name: cred.Name, - Description: cred.Description, - Endpoint: endpoint.Name, - AuthType: params.ForgeAuthType(cred.GetAuthType()), - } - switch credParams.AuthType { - case params.ForgeAuthTypeApp: - keyBytes, err := cred.App.PrivateKeyBytes() - if err != nil { - return fmt.Errorf("error getting private key bytes: %w", err) - } - credParams.App = params.GithubApp{ - AppID: cred.App.AppID, - InstallationID: cred.App.InstallationID, - PrivateKeyBytes: keyBytes, - } - - if err := credParams.App.Validate(); err != nil { - return fmt.Errorf("error validating app credentials: %w", err) - } - case params.ForgeAuthTypePAT: - token := cred.PAT.OAuth2Token - if token == "" { - token = cred.OAuth2Token - } - if token == "" { - return errors.New("missing OAuth2 token") - } - credParams.PAT = params.GithubPAT{ - OAuth2Token: token, - } - } - - creds, err := s.CreateGithubCredentials(adminCtx, credParams) - if err != nil { - return fmt.Errorf("error creating github credentials: %w", err) - } - - if err := s.conn.Exec("update repositories set credentials_id = ?,endpoint_name = ? where credentials_name = ?", creds.ID, creds.Endpoint.Name, creds.Name).Error; err != nil { - return fmt.Errorf("error updating repositories: %w", err) - } - - if err := s.conn.Exec("update organizations set credentials_id = ?,endpoint_name = ? where credentials_name = ?", creds.ID, creds.Endpoint.Name, creds.Name).Error; err != nil { - return fmt.Errorf("error updating organizations: %w", err) - } - - if err := s.conn.Exec("update enterprises set credentials_id = ?,endpoint_name = ? where credentials_name = ?", creds.ID, creds.Endpoint.Name, creds.Name).Error; err != nil { - return fmt.Errorf("error updating enterprises: %w", err) - } - } - return nil -} - -func (s *sqlDatabase) migrateWorkflow() error { - if s.conn.Migrator().HasTable(&WorkflowJob{}) { - if s.conn.Migrator().HasColumn(&WorkflowJob{}, "runner_name") { - // Remove jobs that are not in "queued" status. We really only care about queued jobs. Once they transition - // to something else, we don't really consume them anyway. - if err := s.conn.Exec("delete from workflow_jobs where status is not 'queued'").Error; err != nil { - return fmt.Errorf("error updating workflow_jobs: %w", err) - } - if err := s.conn.Migrator().DropColumn(&WorkflowJob{}, "runner_name"); err != nil { - return fmt.Errorf("error updating workflow_jobs: %w", err) - } - } - } - return nil -} - func (s *sqlDatabase) migrateFileObjects() error { - // Only migrate for SQLite backend - if s.cfg.DbBackend != config.SQLiteBackend { - return nil - } - - // Use the separate objects database connection if s.objectsConn == nil { - return fmt.Errorf("objects database connection not initialized") - } - - // Use GORM AutoMigrate on the separate connection - if err := s.objectsConn.AutoMigrate(&FileObject{}, &FileBlob{}, &FileObjectTag{}); err != nil { - return fmt.Errorf("failed to migrate file objects: %w", err) - } - - return nil -} - -// migrateAutoVacuum converts SQLite databases from auto_vacuum=NONE to -// auto_vacuum=INCREMENTAL. This is a one-time migration that allows the -// periodic maintenance goroutine to use PRAGMA incremental_vacuum to -// efficiently reclaim free pages without rebuilding the entire database. -func (s *sqlDatabase) migrateAutoVacuum() error { - if s.cfg.DbBackend != config.SQLiteBackend { - return nil - } - - if err := s.enableIncrementalVacuum(s.conn, "main"); err != nil { - return err - } - - if s.objectsConn != nil { - if err := s.enableIncrementalVacuum(s.objectsConn, "objects"); err != nil { - return err - } - } - - return nil -} - -func (s *sqlDatabase) enableIncrementalVacuum(conn *gorm.DB, dbName string) error { - var autoVacuum int - if err := conn.Raw("PRAGMA auto_vacuum").Scan(&autoVacuum).Error; err != nil { - return fmt.Errorf("failed to read auto_vacuum for %s db: %w", dbName, err) - } - - // 2 = INCREMENTAL, already set - if autoVacuum == 2 { return nil } - slog.InfoContext(s.ctx, "migrating database to incremental auto_vacuum", "database", dbName) + m := gormigrate.New(s.objectsConn, gormigrate.DefaultOptions, migrations.AllFileObjects()) + m.InitSchema(func(tx *gorm.DB) error { + return tx.AutoMigrate(&FileObject{}, &FileBlob{}, &FileObjectTag{}) + }) - if err := conn.Exec("PRAGMA auto_vacuum = INCREMENTAL").Error; err != nil { - return fmt.Errorf("failed to set auto_vacuum for %s db: %w", dbName, err) - } - // VACUUM is required to apply the auto_vacuum mode change. - if err := conn.Exec("VACUUM").Error; err != nil { - return fmt.Errorf("failed to vacuum %s db for auto_vacuum migration: %w", dbName, err) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running file objects migrations: %w", err) } return nil @@ -535,8 +225,7 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error { if !migrateTemplates { return nil } - // make sure we have a default forge/OSType template. Currently we have Windows - // and Linux for GitHub and Linux for Gitea. + // make sure we have a default forge/OSType template. githubWindowsData, err := templates.GetTemplateContent(commonParams.Windows, params.GithubEndpointType) if err != nil { return fmt.Errorf("failed to get windows template for github: %w", err) @@ -567,8 +256,7 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error { Data: githubWindowsData, IsSystem: true, } - githubWindowsSystemTemplate, err := s.CreateTemplate(adminCtx, githubWindowsParams) - if err != nil { + if _, err := s.CreateTemplate(adminCtx, githubWindowsParams); err != nil { return fmt.Errorf("failed to create github windows template: %w", err) } @@ -580,8 +268,7 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error { Data: githubLinuxData, IsSystem: true, } - githubLinuxSystemTemplate, err := s.CreateTemplate(adminCtx, githubLinuxParams) - if err != nil { + if _, err := s.CreateTemplate(adminCtx, githubLinuxParams); err != nil { return fmt.Errorf("failed to create github linux template: %w", err) } @@ -593,8 +280,7 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error { Data: giteaLinuxData, IsSystem: true, } - giteaLinuxSystemTemplate, err := s.CreateTemplate(adminCtx, giteaLinuxParams) - if err != nil { + if _, err := s.CreateTemplate(adminCtx, giteaLinuxParams); err != nil { return fmt.Errorf("failed to create gitea linux template: %w", err) } @@ -606,225 +292,15 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error { Data: giteaWindowsData, IsSystem: true, } - giteaWindowsSystemTemplate, err := s.CreateTemplate(adminCtx, giteaWindowsParams) - if err != nil { + if _, err := s.CreateTemplate(adminCtx, giteaWindowsParams); err != nil { return fmt.Errorf("failed to create gitea windows template: %w", err) } - getTplID := func(forgeType params.EndpointType, osType commonParams.OSType) uint { - var templateID uint - switch forgeType { - case params.GiteaEndpointType: - switch osType { - case commonParams.Linux: - templateID = giteaLinuxSystemTemplate.ID - case commonParams.Windows: - templateID = giteaWindowsSystemTemplate.ID - default: - return 0 - } - case params.GithubEndpointType: - switch osType { - case commonParams.Linux: - templateID = githubLinuxSystemTemplate.ID - case commonParams.Windows: - templateID = githubWindowsSystemTemplate.ID - default: - return 0 - } - default: - return 0 - } - return templateID - } - - pools, err := s.ListAllPools(s.ctx) - if err != nil { - return fmt.Errorf("failed to list pools: %w", err) - } - - for _, pool := range pools { - forgeType := pool.Endpoint.EndpointType - osType := pool.OSType - entity, err := pool.GetEntity() - if err != nil { - return fmt.Errorf("failed to get pool entity: %w", err) - } - templateID := getTplID(forgeType, osType) - if pool.TemplateID == 0 && templateID != 0 { - updateParams := params.UpdatePoolParams{ - TemplateID: &templateID, - } - if _, err := s.UpdateEntityPool(adminCtx, entity, pool.ID, updateParams); err != nil { - return fmt.Errorf("failed to update pool template: %w", err) - } - } - } - - scaleSets, err := s.ListAllScaleSets(adminCtx) - if err != nil { - return fmt.Errorf("failed to list scale sets: %w", err) - } - - for _, scaleSet := range scaleSets { - forgeType := scaleSet.Endpoint.EndpointType - osType := scaleSet.OSType - entity, err := scaleSet.GetEntity() - if err != nil { - return fmt.Errorf("failed to get scale set entity: %w", err) - } - templateID := getTplID(forgeType, osType) - if scaleSet.TemplateID == 0 && templateID != 0 { - updateParams := params.UpdateScaleSetParams{ - TemplateID: &templateID, - } - if _, err := s.UpdateEntityScaleSet(adminCtx, entity, scaleSet.ID, updateParams, nil); err != nil { - return fmt.Errorf("failed to update pool template: %w", err) - } - } - } - - return nil -} - -// dropIndexIfExists drops an index if it exists -func (s *sqlDatabase) dropIndexIfExists(model interface{}, indexName string) { - if s.conn.Migrator().HasIndex(model, indexName) { - if err := s.conn.Migrator().DropIndex(model, indexName); err != nil { - slog.With(slog.Any("error", err)). - Error(fmt.Sprintf("failed to drop index %s", indexName)) - } - } -} - -// migratePoolNullIDs updates pools to set null IDs instead of zero UUIDs -func (s *sqlDatabase) migratePoolNullIDs() error { - if !s.conn.Migrator().HasTable(&Pool{}) { - return nil - } - - zeroUUID := "00000000-0000-0000-0000-000000000000" - updates := []struct { - column string - query string - }{ - {"repo_id", fmt.Sprintf("update pools set repo_id=NULL where repo_id='%s'", zeroUUID)}, - {"org_id", fmt.Sprintf("update pools set org_id=NULL where org_id='%s'", zeroUUID)}, - {"enterprise_id", fmt.Sprintf("update pools set enterprise_id=NULL where enterprise_id='%s'", zeroUUID)}, - } - - for _, update := range updates { - if err := s.conn.Exec(update.query).Error; err != nil { - return fmt.Errorf("error updating pools %s: %w", update.column, err) - } - } - return nil -} - -// migrateGithubEndpointType adds and initializes endpoint_type column -func (s *sqlDatabase) migrateGithubEndpointType() error { - if !s.conn.Migrator().HasTable(&GithubEndpoint{}) { - return nil - } - - if s.conn.Migrator().HasColumn(&GithubEndpoint{}, "endpoint_type") { - return nil - } - - if err := s.conn.Migrator().AutoMigrate(&GithubEndpoint{}); err != nil { - return fmt.Errorf("error migrating github endpoints: %w", err) - } - - if err := s.conn.Exec("update github_endpoints set endpoint_type = 'github' where endpoint_type is null").Error; err != nil { - return fmt.Errorf("error updating github endpoints: %w", err) - } - - return nil -} - -// migrateControllerInfo updates controller info with new fields -func (s *sqlDatabase) migrateControllerInfo(hasMinAgeField, hasAgentURL bool) error { - if hasMinAgeField && hasAgentURL { - return nil - } - - var controller ControllerInfo - if err := s.conn.First(&controller).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil - } - return fmt.Errorf("error fetching controller info: %w", err) - } - - if !hasMinAgeField { - controller.MinimumJobAgeBackoff = 30 - } - - if controller.GARMAgentReleasesURL == "" { - controller.GARMAgentReleasesURL = appdefaults.GARMAgentDefaultReleasesURL - } - - if !hasAgentURL && controller.WebhookBaseURL != "" { - matchWebhooksPath := regexp.MustCompile(`/webhooks(/)?$`) - controller.AgentURL = matchWebhooksPath.ReplaceAllLiteralString(controller.WebhookBaseURL, `/agent`) - } - - if err := s.conn.Save(&controller).Error; err != nil { - return fmt.Errorf("error updating controller info: %w", err) - } - return nil } -// preMigrationChecks performs checks before running migrations -func (s *sqlDatabase) preMigrationChecks() (needsCredentialMigration, migrateTemplates, hasMinAgeField, hasAgentURL bool) { - // Check if credentials need migration - needsCredentialMigration = !s.conn.Migrator().HasTable(&GithubCredentials{}) || - !s.conn.Migrator().HasTable(&GithubEndpoint{}) - - // Check if templates need migration - migrateTemplates = !s.conn.Migrator().HasTable(&Template{}) - - // Check for controller info fields - if s.conn.Migrator().HasTable(&ControllerInfo{}) { - hasMinAgeField = s.conn.Migrator().HasColumn(&ControllerInfo{}, "minimum_job_age_backoff") - hasAgentURL = s.conn.Migrator().HasColumn(&ControllerInfo{}, "agent_url") - } - - return -} - -func (s *sqlDatabase) migrateDB() error { - // Drop obsolete indexes - s.dropIndexIfExists(&Organization{}, "idx_organizations_name") - s.dropIndexIfExists(&Repository{}, "idx_owner") - - // Run cascade migration - if err := s.cascadeMigration(); err != nil { - return fmt.Errorf("error running cascade migration: %w", err) - } - - // Migrate pool null IDs - if err := s.migratePoolNullIDs(); err != nil { - return err - } - - // Migrate workflows - if err := s.migrateWorkflow(); err != nil { - return fmt.Errorf("error migrating workflows: %w", err) - } - - // Migrate GitHub endpoint type - if err := s.migrateGithubEndpointType(); err != nil { - return err - } - - // Check if we need to migrate credentials and templates - needsCredentialMigration, migrateTemplates, hasMinAgeField, hasAgentURL := s.preMigrationChecks() - - // Run main schema migration - s.conn.Exec("PRAGMA foreign_keys = OFF") - if err := s.conn.AutoMigrate( +func (s *sqlDatabase) initSchema(tx *gorm.DB) error { + if err := tx.AutoMigrate( &User{}, &GithubEndpoint{}, &GithubCredentials{}, @@ -847,38 +323,30 @@ func (s *sqlDatabase) migrateDB() error { ); err != nil { return fmt.Errorf("error running auto migrate: %w", err) } + return nil +} - // Migrate file object tables in the attached objectsdb schema - if err := s.migrateFileObjects(); err != nil { - return fmt.Errorf("error migrating file objects: %w", err) - } +func (s *sqlDatabase) migrateDB() error { + m := gormigrate.New(s.conn, gormigrate.DefaultOptions, migrations.All()) + m.InitSchema(s.initSchema) - // Migrate auto_vacuum mode to incremental for both databases. - if err := s.migrateAutoVacuum(); err != nil { - return fmt.Errorf("error migrating auto_vacuum: %w", err) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error running migrations: %w", err) } - s.conn.Exec("PRAGMA foreign_keys = ON") - - // Migrate controller info if needed - if err := s.migrateControllerInfo(hasMinAgeField, hasAgentURL); err != nil { - return err + // Migrate file object tables in the separate objects database + if err := s.migrateFileObjects(); err != nil { + return fmt.Errorf("error migrating file objects: %w", err) } - // Ensure github endpoint exists + // Seed default data if err := s.ensureGithubEndpoint(); err != nil { return fmt.Errorf("error ensuring github endpoint: %w", err) } - // Migrate credentials if needed - if needsCredentialMigration { - if err := s.migrateCredentialsToDB(); err != nil { - return fmt.Errorf("error migrating credentials: %w", err) - } - } - - // Ensure templates exist - if err := s.ensureTemplates(migrateTemplates); err != nil { + var tplCount int64 + s.conn.Model(&Template{}).Count(&tplCount) + if err := s.ensureTemplates(tplCount == 0); err != nil { return fmt.Errorf("failed to create default templates: %w", err) } diff --git a/go.mod b/go.mod index 0fe0c49a..1f18a930 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/cloudbase/garm-provider-common v0.1.9 github.com/felixge/httpsnoop v1.0.4 github.com/gdamore/tcell/v2 v2.13.9 + github.com/go-gormigrate/gormigrate/v2 v2.1.5 github.com/go-openapi/errors v0.22.7 github.com/go-openapi/runtime v0.29.5 github.com/go-openapi/strfmt v0.26.2 diff --git a/go.sum b/go.sum index c45535bf..a7feb1cf 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/gdamore/encoding v1.0.1 h1:YzKZckdBL6jVt2Gc+5p82qhrGiqMdG/eNs6Wy0u3Uh github.com/gdamore/encoding v1.0.1/go.mod h1:0Z0cMFinngz9kS1QfMjCP8TY7em3bZYeeklsSDPivEo= github.com/gdamore/tcell/v2 v2.13.9 h1:uI5l3DYPcFvHINKlGft+en23evOKL+dwtD21QR8ejVA= github.com/gdamore/tcell/v2 v2.13.9/go.mod h1:+Wfe208WDdB7INEtCsNrAN6O2m+wsTPk1RAovjaILlo= +github.com/go-gormigrate/gormigrate/v2 v2.1.5 h1:1OyorA5LtdQw12cyJDEHuTrEV3GiXiIhS4/QTTa/SM8= +github.com/go-gormigrate/gormigrate/v2 v2.1.5/go.mod h1:mj9ekk/7CPF3VjopaFvWKN2v7fN3D9d3eEOAXRhi/+M= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/vendor/github.com/go-gormigrate/gormigrate/v2/.gitignore b/vendor/github.com/go-gormigrate/gormigrate/v2/.gitignore new file mode 100644 index 00000000..86671f1b --- /dev/null +++ b/vendor/github.com/go-gormigrate/gormigrate/v2/.gitignore @@ -0,0 +1,5 @@ +# macOS +.DS_Store + +# Docker +.env diff --git a/vendor/github.com/go-gormigrate/gormigrate/v2/CHANGELOG.md b/vendor/github.com/go-gormigrate/gormigrate/v2/CHANGELOG.md new file mode 100644 index 00000000..c4b968f1 --- /dev/null +++ b/vendor/github.com/go-gormigrate/gormigrate/v2/CHANGELOG.md @@ -0,0 +1,113 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## NOTE: changelog deprecation + After 2.1.4, please see changes in GitHub release description. + +## [Unreleased] + +## [2.1.4] - 2025-03-28 +### Changed +- Update dependencies +- Allow use of nil for options (by @ski7777 in #228) + +## [2.1.3] - 2024-09-23 +### Changed +- Update dependencies + +## [2.1.2] - 2024-03-18 +### Added +- Fix LastInsertId error in postgres driver (by @oxyno-zeta) +### Changed +- Update dependencies + +## [2.1.1] - 2023-09-14 +### Added +- Add to test suite pure-go sqlite implementation: github.com/glebarez/sqlite +### Changed +- Update databases for ci pipelines and local development +- Extract integration tests and their deps into different integration-test package +- Refactor raw sql query into native Gorm method chain, for better compatibility with all dialects + +## [2.1.0] - 2023-06-01 +### Changed +- Refactor plain sql mutation statements (create, insert, delete) into native Gorm methods +- Update dependencies + +## [2.0.3] - 2023-05-29 +### Changed +- Update dependencies + +## [2.0.2] - 2022-05-29 +### Changed +- Update dependencies + +## [2.0.1] - 2022-05-15 +### Changed +- Update dependencies + +## [2.0.0] - 2020-09-05 +### Changed +- Make it compatible with Gorm v2, which uses a new import path and has + breaking changes on its API + ([#45](https://github.com/go-gormigrate/gormigrate/issues/45), [#46](https://github.com/go-gormigrate/gormigrate/pull/46)). + +## [1.6.0] - 2019-07-07 +### Added +- Add option to return an error if the database have unknown migrations + (defaults to `false`) + ([#37](https://github.com/go-gormigrate/gormigrate/pull/37)). + +## [1.5.0] - 2019-04-29 +### Changed +- Making the code more safe by checking more errors + ([#35](https://github.com/go-gormigrate/gormigrate/pull/35)). +### Fixed +- Fixed and written tests for transaction handling + ([#34](https://github.com/go-gormigrate/gormigrate/pull/34), [#10](https://github.com/go-gormigrate/gormigrate/issues/10)). + Enabling transation is recommend, but only supported for databases that + support DDL transactions (PostgreSQL, Microsoft SQL Server and SQLite). + +## [1.4.0] - 2019-02-03 +### Changed +- Allow an empty migration list if a `InitSchema` function is defined + ([#28](https://github.com/go-gormigrate/gormigrate/pull/28)). + +## [1.3.1] - 2019-01-26 +### Fixed +- Fixed `testify` import path from `gopkg.in/stretchr/testify.v1` to + `github.com/stretchr/testify` ([#27](https://github.com/go-gormigrate/gormigrate/pull/27)). + +## [1.3.0] - 2018-12-02 +### Changed +- Starting from this release, this package is available as a [Go Module](https://github.com/golang/go/wiki/Modules). + Import path is still `gopkg.in/gormigrate.v1` in this major version, but will + change to `github.com/go-gormigrate/gormigrate/v2` in the next major release; +- Validate the ID exists on the migration list (#20, #21). + +## [1.2.1] - 2018-09-07 +### Added +- Added `MigrateTo` and `RollbackTo` methods (#15); +- CI now runs tests for SQLite, PostgreSQL, MySQL and Microsoft SQL Server. +### Changed +- An empty migration list is not allowed anymore. Please, make sure that you + have at least one migration, even if dummy; + +## [1.2.0] - 2018-07-12 +### Added +- Add `IDColumnSize` options, which defaults to `255` (#7); + +## [1.1.4] - 2018-05-06 +### Changed +- Assuming default options if blank; +- Returning an error if the migration list has a duplicated migration ID. + +## [1.1.3] - 2018-02-25 +### Added +- Introduce changelog +### Fixed +- Fix `RollbackLast` (#4). diff --git a/vendor/github.com/go-gormigrate/gormigrate/v2/LICENSE b/vendor/github.com/go-gormigrate/gormigrate/v2/LICENSE new file mode 100644 index 00000000..50cde21c --- /dev/null +++ b/vendor/github.com/go-gormigrate/gormigrate/v2/LICENSE @@ -0,0 +1,8 @@ +MIT License +Copyright (c) 2016 Andrey Nering + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/go-gormigrate/gormigrate/v2/README.md b/vendor/github.com/go-gormigrate/gormigrate/v2/README.md new file mode 100644 index 00000000..894fabab --- /dev/null +++ b/vendor/github.com/go-gormigrate/gormigrate/v2/README.md @@ -0,0 +1,232 @@ +# Gormigrate + +[![Latest Release](https://img.shields.io/github/release/go-gormigrate/gormigrate.svg)](https://github.com/go-gormigrate/gormigrate/releases) +[![Go Reference](https://pkg.go.dev/badge/github.com/go-gormigrate/gormigrate/v2.svg)](https://pkg.go.dev/github.com/go-gormigrate/gormigrate/v2) +[![Go Report Card](https://goreportcard.com/badge/github.com/go-gormigrate/gormigrate/v2)](https://goreportcard.com/report/github.com/go-gormigrate/gormigrate/v2) +[![CI | Lint](https://github.com/go-gormigrate/gormigrate/actions/workflows/lint.yml/badge.svg)](https://github.com/go-gormigrate/gormigrate/actions) +[![CI | Test](https://github.com/go-gormigrate/gormigrate/actions/workflows/integration-test.yml/badge.svg)](https://github.com/go-gormigrate/gormigrate/actions) + +Gormigrate is a minimalistic migration helper for [Gorm](http://gorm.io). +Gorm already has useful [migrate functions](https://gorm.io/docs/migration.html), just misses +proper schema versioning and migration rollback support. + +> IMPORTANT: If you need support to Gorm v1 (which uses +> `github.com/jinzhu/gorm` as its import path), please import Gormigrate by +> using the `gopkg.in/gormigrate.v1` import path. +> +> The current Gorm version (v2) is supported by using the +> `github.com/go-gormigrate/gormigrate/v2` import path as described in the +> documentation below. + +## Supported databases + +It supports any of the [databases Gorm supports](https://gorm.io/docs/connecting_to_the_database.html): + +- MySQL +- MariaDB +- PostgreSQL +- SQLite +- Microsoft SQL Server +- TiDB +- Clickhouse + +## Usage + +```go +package main + +import ( + "log" + + "github.com/go-gormigrate/gormigrate/v2" + "github.com/google/uuid" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func main() { + db, err := gorm.Open(sqlite.Open("./data.db"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Info), + }) + if err != nil { + log.Fatal(err) + } + + m := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{{ + // create `users` table + ID: "201608301400", + Migrate: func(tx *gorm.DB) error { + // it's a good pratice to copy the struct inside the function, + // so side effects are prevented if the original struct changes during the time + type user struct { + ID uuid.UUID `gorm:"type:uuid;primaryKey;uniqueIndex"` + Name string + } + return tx.Migrator().CreateTable(&user{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("users") + }, + }, { + // add `age` column to `users` table + ID: "201608301415", + Migrate: func(tx *gorm.DB) error { + // when table already exists, define only columns that are about to change + type user struct { + Age int + } + return tx.Migrator().AddColumn(&user{}, "Age") + }, + Rollback: func(tx *gorm.DB) error { + type user struct { + Age int + } + return db.Migrator().DropColumn(&user{}, "Age") + }, + }, { + // create `organizations` table where users belong to + ID: "201608301430", + Migrate: func(tx *gorm.DB) error { + type organization struct { + ID uuid.UUID `gorm:"type:uuid;primaryKey;uniqueIndex"` + Name string + Address string + } + if err := tx.Migrator().CreateTable(&organization{}); err != nil { + return err + } + type user struct { + OrganizationID uuid.UUID `gorm:"type:uuid"` + } + return tx.Migrator().AddColumn(&user{}, "OrganizationID") + }, + Rollback: func(tx *gorm.DB) error { + type user struct { + OrganizationID uuid.UUID `gorm:"type:uuid"` + } + if err := db.Migrator().DropColumn(&user{}, "OrganizationID"); err != nil { + return err + } + return tx.Migrator().DropTable("organizations") + }, + }}) + + if err := m.Migrate(); err != nil { + log.Fatalf("Migration failed: %v", err) + } + log.Println("Migration did run successfully") +} +``` + +## Having a separate function for initializing the schema + +If you have a lot of migrations, it can be a pain to run all them, as example, +when you are deploying a new instance of the app, in a clean database. +To prevent this, you can set a function that will run if no migration was run +before (in a new clean database). Remember to create everything here, all tables, +foreign keys and what more you need in your app. + +```go +type Organization struct { + gorm.Model + Name string + Address string +} + +type User struct { + gorm.Model + Name string + Age int + OrganizationID uint +} + +m := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{ + // your migrations here +}) + +m.InitSchema(func(tx *gorm.DB) error { + err := tx.AutoMigrate( + &Organization{}, + &User{}, + // all other tables of you app + ) + if err != nil { + return err + } + + if err := tx.Exec("ALTER TABLE users ADD CONSTRAINT fk_users_organizations FOREIGN KEY (organization_id) REFERENCES organizations (id)").Error; err != nil { + return err + } + // all other constraints, indexes, etc... + return nil +}) +``` + +## Options + +This is the options struct, in case you don't want the defaults: + +```go +type Options struct { + // TableName is the migration table. + TableName string + // IDColumnName is the name of column where the migration id will be stored. + IDColumnName string + // IDColumnSize is the length of the migration id column + IDColumnSize int + // UseTransaction makes Gormigrate execute migrations inside a single transaction. + // Keep in mind that not all databases support DDL commands inside transactions. + UseTransaction bool + // ValidateUnknownMigrations will cause migrate to fail if there's unknown migration + // IDs in the database + ValidateUnknownMigrations bool +} +``` + +## Who is Gormigrate for? + +Gormigrate was born to be a simple and minimalistic migration tool for small +projects that uses [Gorm](http://gorm.io). You may want to take a look at more advanced +solutions like [golang-migrate/migrate](https://github.com/golang-migrate/migrate) +if you plan to scale. + +Be aware that Gormigrate has no builtin lock mechanism, so if you're running +it automatically and have a distributed setup (i.e. more than one executable +running at the same time), you might want to use a +[distributed lock/mutex mechanism](https://redis.io/topics/distlock) to +prevent race conditions while running migrations. + +## Contributing + +To run integration tests, some preparations are needed. Please ensure you +have [task](https://taskfile.dev/installation) and [docker](https://docs.docker.com/engine/install) installed. +Then: + +1. Ensure target or all databases are available and ready to accept connections. + You can start databases locally with `task docker:compose:up` +2. Copy `integration-test/.example.env` as `integration-test/.env` and + adjust the database connection ports and credentials when needed. +3. Run integration test for single database or for all + +```bash +# run test for MySQL +task test:mysql + +# run test for MariaDB +task test:mariadb + +# run test for PostgreSQL +task test:postgres + +# run test for SQLite +task test:sqlite + +# run test for Microsoft SQL Server +task test:sqlserver + +# run test for all databases +task test:all +``` + +Alternatively, you can run everything in one step: `task docker:test` diff --git a/vendor/github.com/go-gormigrate/gormigrate/v2/Taskfile.yml b/vendor/github.com/go-gormigrate/gormigrate/v2/Taskfile.yml new file mode 100644 index 00000000..c4adbbda --- /dev/null +++ b/vendor/github.com/go-gormigrate/gormigrate/v2/Taskfile.yml @@ -0,0 +1,85 @@ +# https://taskfile.dev + +version: '3' + +tasks: + lint: + desc: Run golangci-lint + cmds: + - golangci-lint run + - cd integration-test && golangci-lint run --path-prefix integration-test + + test: + dir: ./integration-test + cmds: + - go test -v -tags {{.DATABASE}} ./... + + test:mysql: + desc: Run tests for MySQL + cmds: + - task: test + vars: {DATABASE: mysql} + + test:mariadb: + desc: Run tests for MariaDB + cmds: + - task: test + vars: {DATABASE: mariadb} + + test:postgres: + desc: Run tests for PostgreSQL + cmds: + - task: test + vars: {DATABASE: postgres} + + test:sqlite: + desc: Run tests for SQLite + cmds: + - task: test + vars: {DATABASE: sqlite} + + test:sqlitego: + desc: Run tests for SQLite (Pure-Go implementation) + cmds: + - task: test + vars: {DATABASE: sqlitego} + + test:sqlserver: + desc: Run tests for Microsoft SQL Server + cmds: + - task: test + vars: {DATABASE: sqlserver} + + test:all: + desc: Run tests for all databases + deps: [test:mysql, test:mariadb, test:postgres, test:sqlite, test:sqlitego, test:sqlserver] + +# Docker tasks + + docker:compose: + cmds: + - docker compose --project-directory ./integration-test {{.CMD}} + + docker:compose:down: + cmds: + - task: docker:compose + vars: {CMD: down --volumes --remove-orphans} + + docker:compose:up: + desc: Start docker compose with database services + deps: [docker:compose:down] + cmds: + - task: docker:compose + vars: {CMD: up --detach --no-deps --remove-orphans --force-recreate --wait} + + docker:compose:logs: + desc: Start streaming docker compose logs + cmds: + - task: docker:compose + vars: {CMD: logs --follow --tail 50} + + docker:test: + deps: [docker:compose:up] + cmds: + - defer: task docker:compose:down + - task: test:all diff --git a/vendor/github.com/go-gormigrate/gormigrate/v2/gormigrate.go b/vendor/github.com/go-gormigrate/gormigrate/v2/gormigrate.go new file mode 100644 index 00000000..0fd10d6c --- /dev/null +++ b/vendor/github.com/go-gormigrate/gormigrate/v2/gormigrate.go @@ -0,0 +1,493 @@ +// Package gormigrate is a minimalistic migration helper for Gorm (http://gorm.io) +package gormigrate + +import ( + "context" + "errors" + "fmt" + "reflect" + + "gorm.io/gorm" +) + +const ( + initSchemaMigrationID = "SCHEMA_INIT" +) + +// MigrateFunc is the func signature for migrating. +type MigrateFunc func(*gorm.DB) error + +// RollbackFunc is the func signature for rollbacking. +type RollbackFunc func(*gorm.DB) error + +// InitSchemaFunc is the func signature for initializing the schema. +type InitSchemaFunc func(*gorm.DB) error + +// Options define options for all migrations. +type Options struct { + // TableName is the migration table. + TableName string + // IDColumnName is the name of column where the migration id will be stored. + IDColumnName string + // IDColumnSize is the length of the migration id column + IDColumnSize int + // UseTransaction makes Gormigrate execute migrations inside a single transaction. + // Keep in mind that not all databases support DDL commands inside transactions. + UseTransaction bool + // ValidateUnknownMigrations will cause migrate to fail if there's unknown migration + // IDs in the database + ValidateUnknownMigrations bool +} + +// Migration represents a database migration (a modification to be made on the database). +type Migration struct { + // ID is the migration identifier. Usually a timestamp like "201601021504". + ID string + // Migrate is a function that will br executed while running this migration. + Migrate MigrateFunc + // Rollback will be executed on rollback. Can be nil. + Rollback RollbackFunc +} + +// Gormigrate represents a collection of all migrations of a database schema. +type Gormigrate struct { + db *gorm.DB + tx *gorm.DB + options *Options + migrations []*Migration + initSchema InitSchemaFunc +} + +// ReservedIDError is returned when a migration is using a reserved ID +type ReservedIDError struct { + ID string +} + +func (e *ReservedIDError) Error() string { + return fmt.Sprintf(`gormigrate: Reserved migration ID: "%s"`, e.ID) +} + +// DuplicatedIDError is returned when more than one migration have the same ID +type DuplicatedIDError struct { + ID string +} + +func (e *DuplicatedIDError) Error() string { + return fmt.Sprintf(`gormigrate: Duplicated migration ID: "%s"`, e.ID) +} + +var ( + // DefaultOptions can be used if you don't want to think about options. + DefaultOptions = &Options{ + TableName: "migrations", + IDColumnName: "id", + IDColumnSize: 255, + UseTransaction: false, + ValidateUnknownMigrations: false, + } + + // ErrRollbackImpossible is returned when trying to rollback a migration + // that has no rollback function. + ErrRollbackImpossible = errors.New("gormigrate: It's impossible to rollback this migration") + + // ErrNoMigrationDefined is returned when no migration is defined. + ErrNoMigrationDefined = errors.New("gormigrate: No migration defined") + + // ErrMissingID is returned when the ID od migration is equal to "" + ErrMissingID = errors.New("gormigrate: Missing ID in migration") + + // ErrNoRunMigration is returned when any run migration was found while + // running RollbackLast + ErrNoRunMigration = errors.New("gormigrate: Could not find last run migration") + + // ErrMigrationIDDoesNotExist is returned when migrating or rolling back to a migration ID that + // does not exist in the list of migrations + ErrMigrationIDDoesNotExist = errors.New("gormigrate: Tried to migrate to an ID that doesn't exist") + + // ErrUnknownPastMigration is returned if a migration exists in the DB that doesn't exist in the code + ErrUnknownPastMigration = errors.New("gormigrate: Found migration in DB that does not exist in code") +) + +// New returns a new Gormigrate. +func New(db *gorm.DB, options *Options, migrations []*Migration) *Gormigrate { + if options == nil { + options = DefaultOptions + } + if options.TableName == "" { + options.TableName = DefaultOptions.TableName + } + if options.IDColumnName == "" { + options.IDColumnName = DefaultOptions.IDColumnName + } + if options.IDColumnSize == 0 { + options.IDColumnSize = DefaultOptions.IDColumnSize + } + return &Gormigrate{ + db: db, + options: options, + migrations: migrations, + } +} + +// InitSchema sets a function that is run if no migration is found. +// The idea is preventing to run all migrations when a new clean database +// is being migrating. In this function you should create all tables and +// foreign key necessary to your application. +func (g *Gormigrate) InitSchema(initSchema InitSchemaFunc) { + g.initSchema = initSchema +} + +// Migrate executes all migrations that did not run yet. +func (g *Gormigrate) Migrate() error { + if !g.hasMigrations() { + return ErrNoMigrationDefined + } + var targetMigrationID string + if len(g.migrations) > 0 { + targetMigrationID = g.migrations[len(g.migrations)-1].ID + } + return g.migrate(targetMigrationID) +} + +// MigrateTo executes all migrations that did not run yet up to the migration that matches `migrationID`. +func (g *Gormigrate) MigrateTo(migrationID string) error { + if err := g.checkIDExist(migrationID); err != nil { + return err + } + return g.migrate(migrationID) +} + +func (g *Gormigrate) migrate(migrationID string) error { + if !g.hasMigrations() { + return ErrNoMigrationDefined + } + + if err := g.checkReservedID(); err != nil { + return err + } + + if err := g.checkDuplicatedID(); err != nil { + return err + } + + g.begin() + defer g.rollback() + + if err := g.createMigrationTableIfNotExists(); err != nil { + return err + } + + if g.options.ValidateUnknownMigrations { + unknownMigrations, err := g.unknownMigrationsHaveHappened() + if err != nil { + return err + } + if unknownMigrations { + return ErrUnknownPastMigration + } + } + + if g.initSchema != nil { + canInitializeSchema, err := g.canInitializeSchema() + if err != nil { + return err + } + if canInitializeSchema { + if err := g.runInitSchema(); err != nil { + return err + } + return g.commit() + } + } + + for _, migration := range g.migrations { + if err := g.runMigration(migration); err != nil { + return err + } + if migrationID != "" && migration.ID == migrationID { + break + } + } + return g.commit() +} + +// There are migrations to apply if either there's a defined +// initSchema function or if the list of migrations is not empty. +func (g *Gormigrate) hasMigrations() bool { + return g.initSchema != nil || len(g.migrations) > 0 +} + +// Check whether any migration is using a reserved ID. +// For now there's only have one reserved ID, but there may be more in the future. +func (g *Gormigrate) checkReservedID() error { + for _, m := range g.migrations { + if m.ID == initSchemaMigrationID { + return &ReservedIDError{ID: m.ID} + } + } + return nil +} + +func (g *Gormigrate) checkDuplicatedID() error { + lookup := make(map[string]struct{}, len(g.migrations)) + for _, m := range g.migrations { + if _, ok := lookup[m.ID]; ok { + return &DuplicatedIDError{ID: m.ID} + } + lookup[m.ID] = struct{}{} + } + return nil +} + +func (g *Gormigrate) checkIDExist(migrationID string) error { + for _, migrate := range g.migrations { + if migrate.ID == migrationID { + return nil + } + } + return ErrMigrationIDDoesNotExist +} + +// RollbackLast undo the last migration +func (g *Gormigrate) RollbackLast() error { + if len(g.migrations) == 0 { + return ErrNoMigrationDefined + } + + g.begin() + defer g.rollback() + + lastRunMigration, err := g.getLastRunMigration() + if err != nil { + return err + } + + if err := g.rollbackMigration(lastRunMigration); err != nil { + return err + } + return g.commit() +} + +// RollbackTo undoes migrations up to the given migration that matches the `migrationID`. +// Migration with the matching `migrationID` is not rolled back. +func (g *Gormigrate) RollbackTo(migrationID string) error { + if len(g.migrations) == 0 { + return ErrNoMigrationDefined + } + + if err := g.checkIDExist(migrationID); err != nil { + return err + } + + g.begin() + defer g.rollback() + + for i := len(g.migrations) - 1; i >= 0; i-- { + migration := g.migrations[i] + if migration.ID == migrationID { + break + } + migrationRan, err := g.migrationRan(migration) + if err != nil { + return err + } + if migrationRan { + if err := g.rollbackMigration(migration); err != nil { + return err + } + } + } + return g.commit() +} + +func (g *Gormigrate) getLastRunMigration() (*Migration, error) { + for i := len(g.migrations) - 1; i >= 0; i-- { + migration := g.migrations[i] + + migrationRan, err := g.migrationRan(migration) + if err != nil { + return nil, err + } + + if migrationRan { + return migration, nil + } + } + return nil, ErrNoRunMigration +} + +// RollbackMigration undo a migration. +func (g *Gormigrate) RollbackMigration(m *Migration) error { + g.begin() + defer g.rollback() + + if err := g.rollbackMigration(m); err != nil { + return err + } + return g.commit() +} + +func (g *Gormigrate) rollbackMigration(m *Migration) error { + if m.Rollback == nil { + return ErrRollbackImpossible + } + + if err := m.Rollback(g.tx); err != nil { + return err + } + + cond := fmt.Sprintf("%s = ?", g.options.IDColumnName) + return g.tx.Table(g.options.TableName).Where(cond, m.ID).Delete(g.model()).Error +} + +func (g *Gormigrate) runInitSchema() error { + if err := g.initSchema(g.tx); err != nil { + return err + } + if err := g.insertMigration(initSchemaMigrationID); err != nil { + return err + } + + for _, migration := range g.migrations { + if err := g.insertMigration(migration.ID); err != nil { + return err + } + } + + return nil +} + +func (g *Gormigrate) runMigration(migration *Migration) error { + if len(migration.ID) == 0 { + return ErrMissingID + } + + migrationRan, err := g.migrationRan(migration) + if err != nil { + return err + } + if !migrationRan { + if err := migration.Migrate(g.tx); err != nil { + return err + } + + if err := g.insertMigration(migration.ID); err != nil { + return err + } + } + return nil +} + +// model returns pointer to dynamically created gorm migration model struct value +// +// struct defined as { +// ID string `gorm:"primaryKey;column:;size:"` +// } +func (g *Gormigrate) model() any { + f := reflect.StructField{ + Name: reflect.ValueOf("ID").Interface().(string), + Type: reflect.TypeOf(""), + Tag: reflect.StructTag(fmt.Sprintf( + `gorm:"primaryKey;column:%s;size:%d"`, + g.options.IDColumnName, + g.options.IDColumnSize, + )), + } + structType := reflect.StructOf([]reflect.StructField{f}) + structValue := reflect.New(structType).Elem() + return structValue.Addr().Interface() +} + +func (g *Gormigrate) createMigrationTableIfNotExists() error { + if g.tx.Migrator().HasTable(g.options.TableName) { + return nil + } + return g.tx.Table(g.options.TableName).AutoMigrate(g.model()) +} + +func (g *Gormigrate) migrationRan(m *Migration) (bool, error) { + var count int64 + err := g.tx. + Table(g.options.TableName). + Where(fmt.Sprintf("%s = ?", g.options.IDColumnName), m.ID). + Count(&count). + Error + return count > 0, err +} + +// The schema can be initialised only if it hasn't been initialised yet +// and no other migration has been applied already. +func (g *Gormigrate) canInitializeSchema() (bool, error) { + migrationRan, err := g.migrationRan(&Migration{ID: initSchemaMigrationID}) + if err != nil { + return false, err + } + if migrationRan { + return false, nil + } + + // If the ID doesn't exist, we also want the list of migrations to be empty + var count int64 + err = g.tx. + Table(g.options.TableName). + Count(&count). + Error + return count == 0, err +} + +func (g *Gormigrate) unknownMigrationsHaveHappened() (bool, error) { + rows, err := g.tx.Table(g.options.TableName).Select(g.options.IDColumnName).Rows() + if err != nil { + return false, err + } + defer func() { + if err := rows.Close(); err != nil { + g.tx.Logger.Error(context.TODO(), err.Error()) + } + }() + + validIDSet := make(map[string]struct{}, len(g.migrations)+1) + validIDSet[initSchemaMigrationID] = struct{}{} + for _, migration := range g.migrations { + validIDSet[migration.ID] = struct{}{} + } + + for rows.Next() { + var pastMigrationID string + if err := rows.Scan(&pastMigrationID); err != nil { + return false, err + } + if _, ok := validIDSet[pastMigrationID]; !ok { + return true, nil + } + } + + return false, nil +} + +func (g *Gormigrate) insertMigration(id string) error { + record := g.model() + reflect.ValueOf(record).Elem().FieldByName("ID").SetString(id) + return g.tx.Table(g.options.TableName).Create(record).Error +} + +func (g *Gormigrate) begin() { + if g.options.UseTransaction { + g.tx = g.db.Begin() + } else { + g.tx = g.db + } +} + +func (g *Gormigrate) commit() error { + if g.options.UseTransaction { + return g.tx.Commit().Error + } + return nil +} + +func (g *Gormigrate) rollback() { + if g.options.UseTransaction { + g.tx.Rollback() + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b43d43d3..2f1572ee 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -74,6 +74,9 @@ github.com/gdamore/tcell/v2/terminfo/x/xfce github.com/gdamore/tcell/v2/terminfo/x/xterm github.com/gdamore/tcell/v2/terminfo/x/xterm_ghostty github.com/gdamore/tcell/v2/terminfo/x/xterm_kitty +# github.com/go-gormigrate/gormigrate/v2 v2.1.5 +## explicit; go 1.18 +github.com/go-gormigrate/gormigrate/v2 # github.com/go-logr/logr v1.4.3 ## explicit; go 1.18 github.com/go-logr/logr From 1dbe99c73ce7ec7c2c07f091cb330b54d89e6d79 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Sun, 10 May 2026 15:21:58 +0300 Subject: [PATCH 2/2] Pin max connextions to 1 and fix tests Signed-off-by: Gabriel Adrian Samfira --- config/config_test.go | 4 +- database/sql/file_store.go | 97 +++++++++++++----------- database/sql/github_test.go | 144 ------------------------------------ database/sql/instances.go | 2 +- database/sql/sql.go | 13 +++- 5 files changed, 66 insertions(+), 194 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 4cd1c819..75aeca98 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -397,13 +397,13 @@ func TestGormParams(t *testing.T) { dbType, uri, err := cfg.GormParams() require.Nil(t, err) require.Equal(t, SQLiteBackend, dbType) - require.Equal(t, filepath.Join(dir, "garm.db?_journal_mode=WAL&_foreign_keys=ON&_txlock=immediate"), uri) + require.Equal(t, filepath.Join(dir, "garm.db?_journal_mode=WAL&_foreign_keys=ON&_txlock=immediate&_auto_vacuum=incremental"), uri) cfg.SQLite.BusyTimeoutSeconds = 5 dbType, uri, err = cfg.GormParams() require.Nil(t, err) require.Equal(t, SQLiteBackend, dbType) - require.Equal(t, filepath.Join(dir, "garm.db?_journal_mode=WAL&_foreign_keys=ON&_txlock=immediate&_busy_timeout=5000"), uri) + require.Equal(t, filepath.Join(dir, "garm.db?_journal_mode=WAL&_foreign_keys=ON&_txlock=immediate&_auto_vacuum=incremental&_busy_timeout=5000"), uri) cfg.DbBackend = MySQLBackend cfg.MySQL = getMySQLDefaultConfig() diff --git a/database/sql/file_store.go b/database/sql/file_store.go index 9708df4a..f75c2667 100644 --- a/database/sql/file_store.go +++ b/database/sql/file_store.go @@ -20,7 +20,47 @@ import ( "github.com/cloudbase/garm/util" ) -// func (s *sqlDatabase) CreateFileObject(ctx context.Context, name string, size int64, tags []string, reader io.Reader) (fileObjParam params.FileObject, err error) { +// streamBlobContent opens a raw SQLite blob handle, streams initialData followed +// by the rest of r into it, and returns the hex-encoded SHA256 of the written content. +// The raw *sql.Conn is closed before returning so the caller can safely use +// s.objectsConn afterwards without pool starvation. +func (s *sqlDatabase) streamBlobContent(ctx context.Context, blobID uint, initialData []byte, r io.Reader) (string, error) { + conn, err := s.objectsSQLDB.Conn(ctx) + if err != nil { + return "", fmt.Errorf("failed to get connection from pool: %w", err) + } + defer conn.Close() + + var sha256sum string + err = conn.Raw(func(driverConn any) error { + sqliteConn := driverConn.(*sqlite3.SQLiteConn) + + blob, err := sqliteConn.Blob("main", "file_blobs", "content", int64(blobID), 1) + if err != nil { + return fmt.Errorf("failed to open blob: %w", err) + } + defer blob.Close() + + hasher := sha256.New() + + if _, err := blob.Write(initialData); err != nil { + return fmt.Errorf("failed to write blob initial buffer: %w", err) + } + hasher.Write(initialData) + + if _, err := io.Copy(io.MultiWriter(blob, hasher), r); err != nil { + return fmt.Errorf("failed to write blob: %w", err) + } + + sha256sum = hex.EncodeToString(hasher.Sum(nil)) + return nil + }) + if err != nil { + return "", fmt.Errorf("failed to write blob: %w", err) + } + return sha256sum, nil +} + func (s *sqlDatabase) CreateFileObject(ctx context.Context, param params.CreateFileObjectParams, reader io.Reader) (fileObjParam params.FileObject, err error) { // Save the file to temporary storage first. This allows us to accept the entire file, even over // a slow connection, without locking the database as we stream the file to the DB. @@ -99,44 +139,12 @@ func (s *sqlDatabase) CreateFileObject(ctx context.Context, param params.CreateF if err != nil { return params.FileObject{}, fmt.Errorf("failed to create database entry for blob: %w", err) } - // Stream file to blob and compute SHA256 - conn, err := s.objectsSQLDB.Conn(ctx) - if err != nil { - return params.FileObject{}, fmt.Errorf("failed to get connection from pool: %w", err) - } - defer conn.Close() - - var sha256sum string - err = conn.Raw(func(driverConn any) error { - sqliteConn := driverConn.(*sqlite3.SQLiteConn) - - blob, err := sqliteConn.Blob("main", "file_blobs", "content", int64(fileBlob.ID), 1) - if err != nil { - return fmt.Errorf("failed to open blob: %w", err) - } - defer blob.Close() - - // Create SHA256 hasher - hasher := sha256.New() - - // Write the buffered data first - if _, err := blob.Write(buffer[:n]); err != nil { - return fmt.Errorf("failed to write blob initial buffer: %w", err) - } - hasher.Write(buffer[:n]) - - // Stream the rest with hash computation - _, err = io.Copy(io.MultiWriter(blob, hasher), tmpFile) - if err != nil { - return fmt.Errorf("failed to write blob: %w", err) - } - - // Get final hash - sha256sum = hex.EncodeToString(hasher.Sum(nil)) - return nil - }) + // Stream file to blob and compute SHA256. + // We obtain a raw *sql.Conn for the SQLite blob API, which pins a connection + // from the pool. We must close it before using s.objectsConn again. + sha256sum, err := s.streamBlobContent(ctx, fileBlob.ID, buffer[:n], tmpFile) if err != nil { - return params.FileObject{}, fmt.Errorf("failed to write blob: %w", err) + return params.FileObject{}, err } // Update document with SHA256 @@ -405,15 +413,18 @@ func (s *sqlDatabase) SearchFileObjectByTags(_ context.Context, tags []string, p // OpenFileObjectContent opens a blob for reading and returns an io.ReadCloser func (s *sqlDatabase) OpenFileObjectContent(ctx context.Context, objID uint) (io.ReadCloser, error) { - conn, err := s.objectsSQLDB.Conn(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get connection: %w", err) - } - + // Query the blob metadata first, before pinning a raw connection. + // With MaxOpenConns(1), pinning the connection before this query would + // deadlock because GORM needs the same pooled connection. var fileBlob FileBlob if err := s.objectsConn.Where("file_object_id = ?", objID).Omit("content").First(&fileBlob).Error; err != nil { return nil, fmt.Errorf("failed to get file blob: %w", err) } + + conn, err := s.objectsSQLDB.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get connection: %w", err) + } var blobReader io.ReadCloser err = conn.Raw(func(driverConn any) error { sqliteConn := driverConn.(*sqlite3.SQLiteConn) diff --git a/database/sql/github_test.go b/database/sql/github_test.go index 592b6bc9..d8ee215e 100644 --- a/database/sql/github_test.go +++ b/database/sql/github_test.go @@ -15,9 +15,7 @@ package sql import ( - "bytes" "context" - "encoding/json" "fmt" "os" "testing" @@ -26,7 +24,6 @@ import ( runnerErrors "github.com/cloudbase/garm-provider-common/errors" "github.com/cloudbase/garm/auth" - "github.com/cloudbase/garm/config" "github.com/cloudbase/garm/database/common" "github.com/cloudbase/garm/database/watcher" garmTesting "github.com/cloudbase/garm/internal/testing" @@ -909,144 +906,3 @@ func (s *GithubTestSuite) TestDeleteGithubEndpointFailsWithOrgsReposOrCredential func TestGithubTestSuite(t *testing.T) { suite.Run(t, new(GithubTestSuite)) } - -func TestCredentialsAndEndpointMigration(t *testing.T) { - cfg := garmTesting.GetTestSqliteDBConfig(t) - - // Copy the sample DB - data, err := os.ReadFile("../../testdata/db/v0.1.4/garm.db") - if err != nil { - t.Fatalf("failed to read test data: %s", err) - } - - if cfg.SQLite.DBFile == "" { - t.Fatalf("DB file not set") - } - if err := os.WriteFile(cfg.SQLite.DBFile, data, 0o600); err != nil { - t.Fatalf("failed to write test data: %s", err) - } - - // define some credentials - credentials := []config.Github{ - { - Name: "test-creds", - Description: "test creds", - AuthType: config.GithubAuthTypePAT, - PAT: config.GithubPAT{ - OAuth2Token: "test", - }, - }, - { - Name: "ghes-test", - Description: "ghes creds", - APIBaseURL: testAPIBaseURL, - UploadBaseURL: testUploadBaseURL, - BaseURL: testBaseURL, - AuthType: config.GithubAuthTypeApp, - App: config.GithubApp{ - AppID: 1, - InstallationID: 99, - PrivateKeyPath: "../../testdata/certs/srv-key.pem", - }, - }, - } - // Set the config credentials in the cfg. This is what happens in the main function. - // of GARM as well. - cfg.MigrateCredentials = credentials - - ctx := context.Background() - watcher.InitWatcher(ctx) - defer watcher.CloseWatcher() - - db, err := NewSQLDatabase(ctx, cfg) - if err != nil { - t.Fatalf("failed to create db connection: %s", err) - } - - // We expect that 2 endpoints will exist in the migrated DB and 2 credentials. - ctx = garmTesting.ImpersonateAdminContext(ctx, db, t) - - endpoints, err := db.ListGithubEndpoints(ctx) - if err != nil { - t.Fatalf("failed to list endpoints: %s", err) - } - if len(endpoints) != 2 { - t.Fatalf("expected 2 endpoints, got %d", len(endpoints)) - } - if endpoints[0].Name != defaultGithubEndpoint { - t.Fatalf("expected default endpoint to exist, got %s", endpoints[0].Name) - } - if endpoints[1].Name != "example.com" { - t.Fatalf("expected example.com endpoint to exist, got %s", endpoints[1].Name) - } - if endpoints[1].UploadBaseURL != testUploadBaseURL { - t.Fatalf("expected upload base URL to be %s, got %s", testUploadBaseURL, endpoints[1].UploadBaseURL) - } - if endpoints[1].BaseURL != testBaseURL { - t.Fatalf("expected base URL to be %s, got %s", testBaseURL, endpoints[1].BaseURL) - } - if endpoints[1].APIBaseURL != testAPIBaseURL { - t.Fatalf("expected API base URL to be %s, got %s", testAPIBaseURL, endpoints[1].APIBaseURL) - } - - creds, err := db.ListGithubCredentials(ctx) - if err != nil { - t.Fatalf("failed to list credentials: %s", err) - } - if len(creds) != 2 { - t.Fatalf("expected 2 credentials, got %d", len(creds)) - } - if creds[0].Name != "test-creds" { - t.Fatalf("expected test-creds to exist, got %s", creds[0].Name) - } - if creds[1].Name != "ghes-test" { - t.Fatalf("expected ghes-test to exist, got %s", creds[1].Name) - } - if creds[0].Endpoint.Name != defaultGithubEndpoint { - t.Fatalf("expected test-creds to be associated with default endpoint, got %s", creds[0].Endpoint.Name) - } - if creds[1].Endpoint.Name != "example.com" { - t.Fatalf("expected ghes-test to be associated with example.com endpoint, got %s", creds[1].Endpoint.Name) - } - - if creds[0].AuthType != params.ForgeAuthTypePAT { - t.Fatalf("expected test-creds to have PAT auth type, got %s", creds[0].AuthType) - } - if creds[1].AuthType != params.ForgeAuthTypeApp { - t.Fatalf("expected ghes-test to have App auth type, got %s", creds[1].AuthType) - } - if len(creds[0].CredentialsPayload) == 0 { - t.Fatalf("expected test-creds to have credentials payload, got empty") - } - - var pat params.GithubPAT - if err := json.Unmarshal(creds[0].CredentialsPayload, &pat); err != nil { - t.Fatalf("failed to unmarshal test-creds credentials payload: %s", err) - } - if pat.OAuth2Token != "test" { - t.Fatalf("expected test-creds to have PAT token test, got %s", pat.OAuth2Token) - } - - var app params.GithubApp - if err := json.Unmarshal(creds[1].CredentialsPayload, &app); err != nil { - t.Fatalf("failed to unmarshal ghes-test credentials payload: %s", err) - } - if app.AppID != 1 { - t.Fatalf("expected ghes-test to have app ID 1, got %d", app.AppID) - } - if app.InstallationID != 99 { - t.Fatalf("expected ghes-test to have installation ID 99, got %d", app.InstallationID) - } - if app.PrivateKeyBytes == nil { - t.Fatalf("expected ghes-test to have private key bytes, got nil") - } - - certBundle, err := credentials[1].App.PrivateKeyBytes() - if err != nil { - t.Fatalf("failed to read CA cert bundle: %s", err) - } - - if !bytes.Equal(app.PrivateKeyBytes, certBundle) { - t.Fatalf("expected ghes-test private key to be equal to the CA cert bundle") - } -} diff --git a/database/sql/instances.go b/database/sql/instances.go index c59e979a..d9d3758e 100644 --- a/database/sql/instances.go +++ b/database/sql/instances.go @@ -47,7 +47,7 @@ func (s *sqlDatabase) CreateInstance(ctx context.Context, poolID string, param p return fmt.Errorf("error fetching pool: %w", err) } var cnt int64 - q := s.conn.Model(&Instance{}).Where("pool_id = ?", pool.ID).Count(&cnt) + q := tx.Model(&Instance{}).Where("pool_id = ?", pool.ID).Count(&cnt) if q.Error != nil { return fmt.Errorf("error fetching instance count: %w", q.Error) } diff --git a/database/sql/sql.go b/database/sql/sql.go index bcb19e9a..60d61c63 100644 --- a/database/sql/sql.go +++ b/database/sql/sql.go @@ -22,20 +22,18 @@ import ( "log/slog" "time" + "github.com/go-gormigrate/gormigrate/v2" "gorm.io/driver/mysql" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" - "github.com/go-gormigrate/gormigrate/v2" - - "github.com/cloudbase/garm/database/sql/migrations" - runnerErrors "github.com/cloudbase/garm-provider-common/errors" commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm/auth" "github.com/cloudbase/garm/config" "github.com/cloudbase/garm/database/common" + "github.com/cloudbase/garm/database/sql/migrations" "github.com/cloudbase/garm/database/watcher" "github.com/cloudbase/garm/internal/templates" "github.com/cloudbase/garm/params" @@ -94,6 +92,12 @@ func NewSQLDatabase(ctx context.Context, cfg config.Database) (common.Store, err return nil, fmt.Errorf("failed to get underlying database connection: %w", err) } + // SQLite only supports one concurrent writer per database file. + // Limit the pool to a single connection to prevent deadlocks with _txlock=immediate. + if cfg.DbBackend == config.SQLiteBackend { + sqlDB.SetMaxOpenConns(1) + } + db := &sqlDatabase{ conn: conn, sqlDB: sqlDB, @@ -119,6 +123,7 @@ func NewSQLDatabase(ctx context.Context, cfg config.Database) (common.Store, err if err != nil { return nil, fmt.Errorf("failed to get underlying objects database connection: %w", err) } + objectsSQLDB.SetMaxOpenConns(1) db.objectsConn = objectsConn db.objectsSQLDB = objectsSQLDB }