test: add migration idempotency validation

Author: TylerHillery

.github/workflows/ci.yml

@@ -102,8 +102,7 @@ jobs:
             --restrict-key=test \
             > before.sql
           
-          psql "$DATABASE_URL" -c "TRUNCATE TABLE storage.migrations;"
-          npm run migration:run
+          npm run migration:test-idempotency
           
           pg_dump "$DATABASE_URL" \
             --exclude-table-data=storage.migrations \

migrations/tenant/0040-fix-prefix-race-conditions-optimized.sql

@@ -240,6 +240,10 @@ BEGIN
 END;
 $$;
 
+DROP TRIGGER IF EXISTS objects_delete_cleanup ON storage.objects;
+DROP TRIGGER IF EXISTS prefixes_delete_cleanup ON storage.prefixes;
+DROP TRIGGER IF EXISTS objects_update_cleanup ON storage.objects;
+
 -- Trigger bindings
 CREATE TRIGGER objects_delete_cleanup
     AFTER DELETE ON storage.objects

package.json

@@ -10,6 +10,7 @@
     "format": "prettier -c --write src/**",
     "lint": "prettier -v && prettier -c src/**",
     "migration:run": "tsx src/scripts/migrate-call.ts",
+    "migration:test-idempotency": "tsx src/scripts/test-migration-idempotency.ts",
     "migrations:types": "tsx src/scripts/migrations-types.ts",
     "docs:export": "tsx ./src/scripts/export-docs.ts",
     "test:dummy-data": "tsx -r dotenv/config ./src/test/db/import-dummy-data.ts",

src/internal/database/migrations/migrate.ts

@@ -358,7 +358,7 @@ export async function runMigrationsOnTenant({
 }
 
 export async function resetMigration(options: {
-  tenantId: string
+  tenantId?: string
   untilMigration: keyof typeof DBMigration
   markCompletedTillMigration?: keyof typeof DBMigration
   databaseUrl: string
@@ -447,10 +447,12 @@ export async function resetMigration(options: {
           }
         }
 
-        await updateTenantMigrationsState(options.tenantId, {
-          migration: latestRunMigration,
-          state: TenantMigrationStatus.COMPLETED,
-        })
+        if (options.tenantId) {
+          await updateTenantMigrationsState(options.tenantId, {
+            migration: latestRunMigration,
+            state: TenantMigrationStatus.COMPLETED,
+          })
+        }
 
         await pgClient.query(`COMMIT`)
 

src/internal/database/migrations/types.ts

@@ -1,4 +1,5 @@
 export const DBMigration = {
+  'create-migrations-table': 0,
   initialmigration: 1,
   'search-files-search-function': 2,
   'storage-schema': 3,

src/scripts/migrations-types.ts

@@ -6,17 +6,24 @@ function main() {
   const migrationsPath = path.join(__dirname, '..', '..', 'migrations', 'tenant', '*.sql')
   const files = glob.sync(migrationsPath).sort()
 
-  const migrations = files.map((file, index) => {
+  const migrations = [
+    // this migration is hardcoded by the postgres migrations library
+    {
+      file: 'create-migrations-table',
+      index: 0,
+    },
+  ]
+
+  files.forEach((file, index) => {
     const fileName = file
       .split(path.sep)
       .pop()
       ?.replace(/[0-9]+-/, '')
       .replace('.sql', '')
-
-    return {
-      file: fileName,
+    migrations.push({
+      file: fileName || '',
       index: index + 1,
-    }
+    })
   })
 
   const migrationsEnum = migrations.map((migration) => {

src/scripts/test-migration-idempotency.ts

@@ -0,0 +1,39 @@
+import dotenv from 'dotenv'
+dotenv.config()
+
+import { resetMigration, runMigrationsOnTenant } from '@internal/database/migrations'
+import { DBMigration } from '@internal/database/migrations/types'
+import { getConfig } from '../config'
+
+void (async () => {
+  const { databaseURL, dbMigrationFreezeAt } = getConfig()
+  const migrations = Object.keys(DBMigration) as (keyof typeof DBMigration)[]
+
+  let previousMigration: keyof typeof DBMigration = 'create-migrations-table'
+
+  for (const migration of migrations.slice(1)) {
+    console.log(`Running   migration ${migration}`)
+    await runMigrationsOnTenant({
+      databaseUrl: databaseURL,
+      upToMigration: migration,
+    })
+
+    console.log(`Resetting migration ${migration}`)
+    await resetMigration({
+      databaseUrl: databaseURL,
+      untilMigration: previousMigration,
+    })
+
+    console.log(`Rerunning migration ${migration}`)
+    await runMigrationsOnTenant({
+      databaseUrl: databaseURL,
+      upToMigration: migration,
+    })
+
+    if (dbMigrationFreezeAt === migration) {
+      break
+    }
+
+    previousMigration = migration
+  }
+})()