diff --git a/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs b/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs index 502853fb..14b85abe 100644 --- a/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs +++ b/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs @@ -16,6 +16,10 @@ spec :: E2ESpec spec = do Spec.describe "Package Sets endpoint" do Spec.it "accepts unauthenticated add/upgrade requests" do + -- First publish unsafe-coerce to create the tarball in storage + { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData + _ <- Env.pollJobOrFail publishJobId + -- Now add it to the package set { jobId } <- Client.packageSets Fixtures.packageSetAddRequest job <- Env.pollJobOrFail jobId Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust @@ -47,6 +51,10 @@ spec = do Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust Spec.it "returns existing job for duplicate requests" do + -- First publish unsafe-coerce so the package set request is valid + { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData + _ <- Env.pollJobOrFail publishJobId + -- Now test that duplicate requests return the same job ID { jobId: firstJobId } <- Client.packageSets Fixtures.packageSetAddRequest { jobId: secondJobId } <- Client.packageSets Fixtures.packageSetAddRequest Assert.shouldEqual firstJobId secondJobId diff --git a/app-e2e/src/Test/E2E/Endpoint/Startup.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs new file mode 100644 index 00000000..eb17e02c --- /dev/null +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -0,0 +1,185 @@ +-- | E2E tests for the Scheduler and JobExecutor startup, covering: +-- | - scheduleDailyPublish: Detects new package versions via GitHub tags +-- | - scheduleTransfers: Detects packages that moved to new GitHub locations +-- | - schedulePackageSetUpdates: Detects recent uploads for package set inclusion +-- | - checkIfNewCompiler: Detects new compiler and enqueues matrix jobs +-- | +-- | IMPORTANT: These tests must run BEFORE resetTestState is called, since +-- | the scheduler runs at server startup and creates jobs that would be cleared. +module Test.E2E.Endpoint.Startup (spec) where + +import Registry.App.Prelude + +import Data.Array as Array +import Data.Map as Map +import Registry.API.V1 (Job(..)) +import Registry.Location (Location(..)) +import Registry.Operation (AuthenticatedPackageOperation(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.Test.Assert as Assert +import Registry.Version as Version +import Test.E2E.Support.Client as Client +import Test.E2E.Support.Env (E2ESpec) +import Test.Spec as Spec + +spec :: E2ESpec +spec = do + Spec.describe "scheduleDailyPublish" do + Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do + -- The scheduler runs at server startup and should have already + -- fetched tags for packages in the registry metadata. + -- type-equality has v4.0.1 published but v4.0.2 in tags (per wiremock config) + jobs <- Client.getJobs + + -- Find publish jobs for type-equality + let + isTypeEqualityPublishJob :: Job -> Boolean + isTypeEqualityPublishJob = case _ of + PublishJob { packageName, packageVersion } -> + packageName == unsafeFromRight (PackageName.parse "type-equality") + && packageVersion + == unsafeFromRight (Version.parse "4.0.2") + _ -> false + + typeEqualityJob = Array.find isTypeEqualityPublishJob jobs + + case typeEqualityJob of + Just (PublishJob { payload }) -> do + -- The scheduler determines a compatible compiler by looking at the previous + -- version's dependencies and finding the intersection of their supported compilers. + -- type-equality@4.0.1 has no dependencies, so the scheduler falls back to the + -- lowest compiler from the previous version (0.15.10). + let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") + when (payload.compiler /= expectedCompiler) do + Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler + Just _ -> Assert.fail "Expected PublishJob but got different job type" + Nothing -> do + -- Log what jobs we did find for debugging + let publishJobs = Array.filter isPublishJob jobs + Assert.fail $ "Expected to find a publish job for type-equality@4.0.2 but found " + <> show (Array.length publishJobs) + <> " publish jobs: " + <> show (map formatPublishJob publishJobs) + + Spec.it "does not enqueue jobs for already-published versions" do + jobs <- Client.getJobs + + -- type-equality v4.0.1 is already published, should NOT have a new job + let + isDuplicateJob :: Job -> Boolean + isDuplicateJob = case _ of + PublishJob { packageName, packageVersion } -> + packageName == unsafeFromRight (PackageName.parse "type-equality") + && packageVersion + == unsafeFromRight (Version.parse "4.0.1") + _ -> false + + duplicateJob = Array.find isDuplicateJob jobs + + case duplicateJob of + Nothing -> pure unit -- Good, no duplicate job + Just _ -> Assert.fail "Found unexpected publish job for already-published type-equality@4.0.1" + + Spec.describe "scheduleTransfers" do + Spec.it "enqueues transfer jobs when package location changes" do + -- type-equality metadata says old-owner, but tags point to purescript + jobs <- Client.getJobs + let + isTypeEqualityTransferJob :: Job -> Boolean + isTypeEqualityTransferJob = case _ of + TransferJob { packageName } -> + packageName == unsafeFromRight (PackageName.parse "type-equality") + _ -> false + case Array.find isTypeEqualityTransferJob jobs of + Just (TransferJob { packageName, payload }) -> do + -- Verify packageName + when (packageName /= unsafeFromRight (PackageName.parse "type-equality")) do + Assert.fail $ "Wrong package name: " <> PackageName.print packageName + -- Verify newLocation in payload + case payload.payload of + Transfer { newLocation } -> + case newLocation of + GitHub { owner } -> + when (owner /= "purescript") do + Assert.fail $ "Expected owner 'purescript' but got '" <> owner <> "'" + _ -> Assert.fail "Expected GitHub location" + _ -> Assert.fail "Expected Transfer payload" + Just _ -> Assert.fail "Expected TransferJob but got different job type" + Nothing -> do + let transferJobs = Array.filter isTransferJob jobs + Assert.fail $ "Expected to find a transfer job for 'type-equality' but found " + <> show (Array.length transferJobs) + <> " transfer jobs" + + Spec.describe "schedulePackageSetUpdates" do + Spec.it "enqueues package set update for recent uploads not in set" do + jobs <- Client.getJobs + let packageSetJobs = Array.filter isPackageSetJob jobs + case Array.head packageSetJobs of + Just (PackageSetJob { payload }) -> + case payload of + Operation.PackageSetUpdate { packages } -> + case Map.lookup (unsafeFromRight $ PackageName.parse "type-equality") packages of + Just (Just _) -> pure unit + _ -> Assert.fail "Expected type-equality in package set update" + Just _ -> Assert.fail "Expected PackageSetJob but got different job type" + Nothing -> Assert.fail "Expected package set job to be enqueued" + + Spec.describe "checkIfNewCompiler" do + Spec.it "enqueues matrix jobs for packages with no dependencies when new compiler detected" do + -- The test env has compilers 0.15.10 and 0.15.11 available. + -- prelude@6.0.1 fixture only has compiler 0.15.10 in metadata. + -- So 0.15.11 should be detected as "new" at startup, triggering + -- matrix jobs for packages with no dependencies. + jobs <- Client.getJobs + let + isNewCompilerMatrixJob :: Job -> Boolean + isNewCompilerMatrixJob = case _ of + MatrixJob { compilerVersion } -> + compilerVersion == unsafeFromRight (Version.parse "0.15.11") + _ -> false + + matrixJobs = Array.filter isNewCompilerMatrixJob jobs + + -- Get package names from matrix jobs + matrixPackages = Array.mapMaybe + ( \j -> case j of + MatrixJob { packageName } -> Just packageName + _ -> Nothing + ) + matrixJobs + + -- Should have matrix jobs for packages with no dependencies + -- prelude has no dependencies, so it should get a matrix job + let preludeName = unsafeFromRight (PackageName.parse "prelude") + unless (Array.elem preludeName matrixPackages) do + Assert.fail $ "Expected matrix job for prelude with compiler 0.15.11, found: " + <> show (Array.length matrixJobs) + <> " matrix jobs for packages: " + <> show (map PackageName.print matrixPackages) + +-- | Check if a job is a PublishJob +isPublishJob :: Job -> Boolean +isPublishJob = case _ of + PublishJob _ -> true + _ -> false + +-- | Format a PublishJob for debugging output +formatPublishJob :: Job -> String +formatPublishJob = case _ of + PublishJob { packageName, packageVersion } -> + PackageName.print packageName <> "@" <> Version.print packageVersion + _ -> "" + +-- | Check if a job is a TransferJob +isTransferJob :: Job -> Boolean +isTransferJob = case _ of + TransferJob _ -> true + _ -> false + +-- | Check if a job is a PackageSetJob +isPackageSetJob :: Job -> Boolean +isPackageSetJob = case _ of + PackageSetJob _ -> true + _ -> false diff --git a/app-e2e/src/Test/E2E/Support/Env.purs b/app-e2e/src/Test/E2E/Support/Env.purs index 06c8d47b..ca988893 100644 --- a/app-e2e/src/Test/E2E/Support/Env.purs +++ b/app-e2e/src/Test/E2E/Support/Env.purs @@ -16,6 +16,7 @@ module Test.E2E.Support.Env , resetTestState , resetDatabase , resetGitFixtures + , stashGitFixtures , resetLogs , resetGitHubRequestCache , pollJobOrFail @@ -25,6 +26,7 @@ module Test.E2E.Support.Env , gitStatus , isCleanGitStatus , waitForAllMatrixJobs + , waitForAllPendingJobs , isMatrixJobFor , readMetadata , readManifestIndexEntry @@ -99,6 +101,10 @@ runE2E env = flip runReaderT env -- | Resets: database, git fixtures, storage mock, and logs. resetTestState :: E2E Unit resetTestState = do + -- Wait for any pending jobs to complete before clearing state. + -- This is important because startup jobs (like matrix jobs from new compiler + -- detection) may still be running when this is called. + waitForAllPendingJobs resetDatabase resetGitFixtures WireMock.clearStorageRequests @@ -124,9 +130,10 @@ resetDatabase = do -- | Reset the git fixtures to restore original state. -- | This restores metadata files modified by unpublish/transfer operations. -- | --- | Strategy: Reset the origin repos to their initial-fixture tag (created during --- | setup), then delete the server's scratch git clones. The server will --- | re-clone fresh copies on the next operation, ensuring a clean cache state. +-- | Strategy: Reset the origin repos to the `post-startup` tag if it exists (created +-- | by stashGitFixtures after startup jobs complete), otherwise fall back to the +-- | `initial-fixture` tag. Then delete the server's scratch git clones so the +-- | server will re-clone fresh copies on the next operation. resetGitFixtures :: E2E Unit resetGitFixtures = do { stateDir } <- ask @@ -140,13 +147,41 @@ resetGitFixtures = do deleteGitClones scratchDir where resetOrigin dir = do - void $ gitOrFail [ "reset", "--hard", "initial-fixture" ] dir + -- Try to reset to post-startup tag first, fall back to initial-fixture + tag <- hasTag "post-startup" dir + let targetTag = if tag then "post-startup" else "initial-fixture" + void $ gitOrFail [ "reset", "--hard", targetTag ] dir void $ gitOrFail [ "clean", "-fd" ] dir + hasTag tagName dir = do + result <- liftAff $ Git.gitCLI [ "tag", "-l", tagName ] (Just dir) + pure $ case result of + Right output -> String.contains (String.Pattern tagName) output + Left _ -> false + deleteGitClones scratchDir = do liftAff $ FS.Extra.remove $ Path.concat [ scratchDir, "registry" ] liftAff $ FS.Extra.remove $ Path.concat [ scratchDir, "registry-index" ] +-- | Stash the current git fixtures state by creating a `post-startup` tag. +-- | This should be called after startup jobs (like matrix jobs from new compiler +-- | detection) have completed, so that resetGitFixtures can restore to this +-- | state instead of the initial fixtures. +stashGitFixtures :: E2E Unit +stashGitFixtures = do + fixturesDir <- liftEffect $ Env.lookupRequired Env.repoFixturesDir + let + registryOrigin = Path.concat [ fixturesDir, "purescript", "registry" ] + registryIndexOrigin = Path.concat [ fixturesDir, "purescript", "registry-index" ] + createStashTag registryOrigin + createStashTag registryIndexOrigin + Console.log "Stashed git fixtures at post-startup tag" + where + createStashTag dir = do + -- Delete existing tag if present, then create new one at HEAD + void $ liftAff $ Git.gitCLI [ "tag", "-d", "post-startup" ] (Just dir) + void $ gitOrFail [ "tag", "post-startup" ] dir + -- | Clear server log files for test isolation. -- | Deletes *.log files from the scratch/logs directory but preserves the directory itself. resetLogs :: E2E Unit @@ -246,6 +281,26 @@ waitForAllMatrixJobs pkg = go 120 0 liftAff $ Aff.delay (Milliseconds 1000.0) go (attempts - 1) totalCount +-- | Wait for all pending jobs (of any type) to complete. +-- | Useful for ensuring startup jobs finish before running tests that clear the DB. +waitForAllPendingJobs :: E2E Unit +waitForAllPendingJobs = go 300 -- 5 minutes max + where + go :: Int -> E2E Unit + go 0 = liftAff $ Aff.throwError $ Aff.error "Timed out waiting for all jobs to complete" + go attempts = do + jobs <- Client.getJobs + let + pendingJobs = Array.filter (\j -> isNothing (V1.jobInfo j).finishedAt) jobs + pendingCount = Array.length pendingJobs + if pendingCount == 0 then + pure unit + else do + when (attempts `mod` 30 == 0) do + Console.log $ "Waiting for " <> show pendingCount <> " pending jobs to complete..." + liftAff $ Aff.delay (Milliseconds 1000.0) + go (attempts - 1) + -- | Check if a job is a matrix job for the given package. isMatrixJobFor :: PackageFixture -> Job -> Boolean isMatrixJobFor pkg = case _ of diff --git a/app-e2e/src/Test/E2E/Support/Fixtures.purs b/app-e2e/src/Test/E2E/Support/Fixtures.purs index 7fe0b556..b9b4bfe2 100644 --- a/app-e2e/src/Test/E2E/Support/Fixtures.purs +++ b/app-e2e/src/Test/E2E/Support/Fixtures.purs @@ -5,9 +5,11 @@ module Test.E2E.Support.Fixtures , effect , console , prelude + , unsafeCoerce , effectPublishData , effectPublishDataDifferentLocation , consolePublishData + , unsafeCoercePublishData , failingTransferData , nonexistentTransferData , trusteeAuthenticatedData @@ -99,6 +101,22 @@ consolePublishData = , version: console.version } +-- | Publish data for unsafe-coerce@6.0.0, used by package set tests. +-- | Has no dependencies. Published first to create the tarball before adding to package set. +unsafeCoercePublishData :: Operation.PublishData +unsafeCoercePublishData = + { name: unsafeCoerce.name + , location: Just $ GitHub + { owner: "purescript" + , repo: "purescript-unsafe-coerce" + , subdir: Nothing + } + , ref: "v6.0.0" + , compiler: Utils.unsafeVersion "0.15.10" + , resolutions: Nothing + , version: unsafeCoerce.version + } + -- | Unpublish data for effect@4.0.0, used for publish-then-unpublish tests. effectUnpublishData :: UnpublishData effectUnpublishData = @@ -222,11 +240,11 @@ signTransfer privateKey transferData = do , signature } --- | type-equality@4.0.1 fixture package (exists in registry-index but not in initial package set) -typeEquality :: PackageFixture -typeEquality = { name: Utils.unsafePackageName "type-equality", version: Utils.unsafeVersion "4.0.1" } +-- | unsafe-coerce@6.0.0 fixture package (exists in registry-index but not in package set) +unsafeCoerce :: PackageFixture +unsafeCoerce = { name: Utils.unsafePackageName "unsafe-coerce", version: Utils.unsafeVersion "6.0.0" } --- | Package set request to add type-equality@4.0.1. +-- | Package set request to add unsafe-coerce@6.0.0. -- | This is an unauthenticated request (no signature) since adding packages -- | doesn't require trustee authentication. packageSetAddRequest :: PackageSetUpdateRequest @@ -234,7 +252,7 @@ packageSetAddRequest = let payload = PackageSetUpdate { compiler: Nothing - , packages: Map.singleton typeEquality.name (Just typeEquality.version) + , packages: Map.singleton unsafeCoerce.name (Just unsafeCoerce.version) } rawPayload = JSON.print $ CJ.encode Operation.packageSetOperationCodec payload in diff --git a/app-e2e/src/Test/Main.purs b/app-e2e/src/Test/Main.purs index a5b18d43..5a81fbfd 100644 --- a/app-e2e/src/Test/Main.purs +++ b/app-e2e/src/Test/Main.purs @@ -6,10 +6,11 @@ import Data.Time.Duration (Milliseconds(..)) import Test.E2E.Endpoint.Jobs as Jobs import Test.E2E.Endpoint.PackageSets as PackageSets import Test.E2E.Endpoint.Publish as Publish +import Test.E2E.Endpoint.Startup as Startup import Test.E2E.Endpoint.Transfer as Transfer import Test.E2E.Endpoint.Unpublish as Unpublish import Test.E2E.GitHubIssue as GitHubIssue -import Test.E2E.Support.Env (assertReposClean, mkTestEnv, resetTestState, runE2E) +import Test.E2E.Support.Env (assertReposClean, mkTestEnv, resetTestState, runE2E, stashGitFixtures, waitForAllPendingJobs) import Test.E2E.Workflow as Workflow import Test.Spec (hoistSpec) import Test.Spec as Spec @@ -21,6 +22,19 @@ main :: Effect Unit main = do env <- mkTestEnv runSpecAndExitProcess' config [ consoleReporter ] $ hoistE2E env do + -- The scheduler runs at startup and enqueues a bunch of jobs in the DB, + -- so we need to run these tests without cleaning out the state first + Spec.describe "Startup" Startup.spec + + -- After scheduler tests, wait for startup jobs to complete and stash the + -- git fixtures state. This ensures that subsequent tests can reset to + -- a state where startup jobs (like new compiler matrix jobs) have already + -- updated the metadata. + Spec.describe "Setup" do + Spec.it "waits for startup jobs and stashes fixtures" do + waitForAllPendingJobs + stashGitFixtures + Spec.before_ resetTestState $ Spec.after_ assertReposClean $ Spec.describe "E2E Tests" do Spec.describe "Endpoints" do Spec.describe "Publish" Publish.spec diff --git a/app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json b/app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json new file mode 100644 index 00000000..eb6293c5 --- /dev/null +++ b/app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json @@ -0,0 +1,21 @@ +{ + "name": "purescript-unsafe-coerce", + "homepage": "https://github.com/purescript/purescript-unsafe-coerce", + "license": "BSD-3-Clause", + "repository": { + "type": "git", + "url": "https://github.com/purescript/purescript-unsafe-coerce.git" + }, + "ignore": [ + "**/.*", + "bower_components", + "node_modules", + "output", + "test", + "bower.json", + "package.json" + ], + "devDependencies": { + "purescript-console": "^6.0.0" + } +} diff --git a/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js new file mode 100644 index 00000000..6c7317ae --- /dev/null +++ b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js @@ -0,0 +1,5 @@ +// module Unsafe.Coerce + +export const unsafeCoerce = function (x) { + return x; +}; diff --git a/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs new file mode 100644 index 00000000..c38fd4be --- /dev/null +++ b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs @@ -0,0 +1,26 @@ +module Unsafe.Coerce + ( unsafeCoerce + ) where + +-- | A _highly unsafe_ function, which can be used to persuade the type system that +-- | any type is the same as any other type. When using this function, it is your +-- | (that is, the caller's) responsibility to ensure that the underlying +-- | representation for both types is the same. +-- | +-- | Because this function is extraordinarily flexible, type inference +-- | can greatly suffer. It is highly recommended to define specializations of +-- | this function rather than using it as-is. For example: +-- | +-- | ```purescript +-- | fromBoolean :: Boolean -> Json +-- | fromBoolean = unsafeCoerce +-- | ``` +-- | +-- | This way, you won't have any nasty surprises due to the inferred type being +-- | different to what you expected. +-- | +-- | After the v0.14.0 PureScript release, some of what was accomplished via +-- | `unsafeCoerce` can now be accomplished via `coerce` from +-- | `purescript-safe-coerce`. See that library's documentation for more +-- | context. +foreign import unsafeCoerce :: forall a b. a -> b diff --git a/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz b/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz new file mode 100644 index 00000000..ba7126b6 Binary files /dev/null and b/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz differ diff --git a/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz b/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz new file mode 100644 index 00000000..34451628 Binary files /dev/null and b/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz differ diff --git a/app/fixtures/registry/metadata/prelude.json b/app/fixtures/registry/metadata/prelude.json index 8c14057a..0b404725 100644 --- a/app/fixtures/registry/metadata/prelude.json +++ b/app/fixtures/registry/metadata/prelude.json @@ -7,8 +7,7 @@ "6.0.1": { "bytes": 31129, "compilers": [ - "0.15.10", - "0.15.11" + "0.15.10" ], "hash": "sha256-EbbFV0J5xV0WammfgCv6HRFSK7Zd803kkofE8aEoam0=", "publishedTime": "2022-08-18T20:04:00.000Z", diff --git a/app/fixtures/registry/metadata/type-equality.json b/app/fixtures/registry/metadata/type-equality.json index e51b5261..35c13b75 100644 --- a/app/fixtures/registry/metadata/type-equality.json +++ b/app/fixtures/registry/metadata/type-equality.json @@ -1,6 +1,6 @@ { "location": { - "githubOwner": "purescript", + "githubOwner": "old-owner", "githubRepo": "purescript-type-equality" }, "published": { diff --git a/app/src/App/API.purs b/app/src/App/API.purs index 8ebc66ba..8584dc27 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -195,11 +195,18 @@ packageSetUpdate details = do let changeSet = candidates.accepted <#> maybe Remove Update Log.notice "Attempting to build package set update." - PackageSets.upgradeAtomic latestPackageSet (fromMaybe prevCompiler payload.compiler) changeSet >>= case _ of - Left error -> - Except.throw $ "The package set produced from this suggested update does not compile:\n\n" <> error - Right packageSet -> do - let commitMessage = PackageSets.commitMessage latestPackageSet changeSet (un PackageSet packageSet).version + PackageSets.upgradeSequential latestPackageSet (fromMaybe prevCompiler payload.compiler) changeSet >>= case _ of + Nothing -> + Except.throw "No packages could be added to the package set. All packages failed to compile." + Just { failed, succeeded, result: packageSet } -> do + unless (Map.isEmpty failed) do + let + formatFailed = String.joinWith "\n" $ Array.catMaybes $ flip map (Map.toUnfoldable failed) \(Tuple name change) -> + case change of + PackageSets.Update version -> Just $ " - " <> formatPackageVersion name version + PackageSets.Remove -> Nothing + Log.warn $ "Some packages could not be added to the set:\n" <> formatFailed + let commitMessage = PackageSets.commitMessage latestPackageSet succeeded (un PackageSet packageSet).version Registry.writePackageSet packageSet commitMessage Log.notice "Built and released a new package set! Now mirroring to the package-sets repo..." Registry.mirrorPackageSet packageSet diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index 48fbdf4a..4d34f51e 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -18,10 +18,13 @@ import Data.Set as Set import Data.String as String import Data.Time.Duration as Duration import Effect.Aff as Aff +import Effect.Aff.AVar (AVar) +import Effect.Aff.AVar as AVar import Effect.Ref as Ref import JSON as JSON import Node.FS.Aff as FS.Aff import Node.Path as Path +import Registry.Foreign.FSExtra as FS.Extra import Registry.App.CLI.Git (GitResult) import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache (class MemoryEncodable, Cache, CacheRef, MemoryEncoding(..)) @@ -155,6 +158,103 @@ data RepoKey | ManifestIndexRepo | LegacyPackageSetsRepo +derive instance Eq RepoKey +derive instance Ord RepoKey + +-- | Identifies which process is using the registry, for lock ownership tracking. +data Process + = Scheduler + | JobExecutor + | API + | ScriptLegacyImporter + | ScriptPackageDeleter + | ScriptSolver + | ScriptVerifyIntegrity + | ScriptCompilerVersions + | ScriptArchiveSeeder + +derive instance Eq Process + +instance Show Process where + show Scheduler = "Scheduler" + show JobExecutor = "JobExecutor" + show API = "API" + show ScriptLegacyImporter = "ScriptLegacyImporter" + show ScriptPackageDeleter = "ScriptPackageDeleter" + show ScriptSolver = "ScriptSolver" + show ScriptVerifyIntegrity = "ScriptVerifyIntegrity" + show ScriptCompilerVersions = "ScriptCompilerVersions" + show ScriptArchiveSeeder = "ScriptArchiveSeeder" + +-- | A lock for a single repository, tracking both the mutex and the owner. +type RepoLock = { lock :: AVar Unit, owner :: Ref (Maybe Process) } + +-- | Per-repository locks to prevent concurrent access. +type RepoLocks = Ref (Map RepoKey RepoLock) + +-- | Create a new empty set of repo locks. +newRepoLocks :: forall m. MonadEffect m => m RepoLocks +newRepoLocks = liftEffect $ Ref.new Map.empty + +-- | Get or create a lock for a repository. +getOrCreateLock :: RepoLocks -> RepoKey -> Aff RepoLock +getOrCreateLock locksRef key = do + locks <- liftEffect $ Ref.read locksRef + case Map.lookup key locks of + Just lock -> pure lock + Nothing -> do + lock <- AVar.new unit + owner <- liftEffect $ Ref.new Nothing + let repoLock = { lock, owner } + liftEffect $ Ref.modify_ (Map.insert key repoLock) locksRef + pure repoLock + +-- | Acquire a repository lock, run an action, and release the lock. +-- | The lock prevents concurrent access to the same repository. +withRepoLock + :: forall r a + . Process + -> RepoLocks + -> RepoKey + -> Run (LOG + AFF + EFFECT + r) a + -> Run (LOG + AFF + EFFECT + r) a +withRepoLock process locks key action = do + repoLock <- Run.liftAff $ getOrCreateLock locks key + Run.liftAff $ AVar.take repoLock.lock + Run.liftEffect $ Ref.write (Just process) repoLock.owner + result <- action + Run.liftEffect $ Ref.write Nothing repoLock.owner + Run.liftAff $ AVar.put unit repoLock.lock + pure result + +-- | Clear any locks owned by a specific process. +-- | Used to clean up orphaned locks when a process crashes and restarts. +clearOwnLocks :: forall r. Process -> RepoLocks -> Run (LOG + AFF + EFFECT + r) Unit +clearOwnLocks process locksRef = do + locks <- Run.liftEffect $ Ref.read locksRef + for_ (Map.toUnfoldable locks :: Array _) \(Tuple _ repoLock) -> do + owner <- Run.liftEffect $ Ref.read repoLock.owner + when (owner == Just process) do + Log.warn $ "Clearing orphaned lock for " <> show process + Run.liftEffect $ Ref.write Nothing repoLock.owner + -- Put the unit back to release the lock + Run.liftAff $ AVar.put unit repoLock.lock + +-- | Validate that a repository is in a valid state. +-- | If the repo is corrupted (e.g., from an interrupted clone), delete it. +validateRepo :: forall r. FilePath -> Run (LOG + AFF + EFFECT + r) Unit +validateRepo path = do + exists <- Run.liftAff $ Aff.attempt (FS.Aff.stat path) + case exists of + Left _ -> pure unit -- Doesn't exist, nothing to validate + Right _ -> do + result <- Run.liftAff $ Git.gitCLI [ "rev-parse", "--git-dir" ] (Just path) + case result of + Left _ -> do + Log.warn $ "Detected corrupted repo at " <> path <> ", deleting" + Run.liftAff $ FS.Extra.remove path + Right _ -> pure unit + -- | A legend for values that can be committed. We know where each kind of value -- | ought to exist, so we can create a correct path for any given type ourselves. data CommitKey @@ -207,6 +307,8 @@ type RegistryEnv = , write :: WriteMode , debouncer :: Debouncer , cacheRef :: CacheRef + , repoLocks :: RepoLocks + , process :: Process } type Debouncer = Ref (Map FilePath DateTime) @@ -705,8 +807,9 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << -- | Get the repository at the given key, recording whether the pull or clone -- | had any effect (ie. if the repo was already up-to-date). + -- | Uses per-repository locking to prevent race conditions during clone. pull :: RepoKey -> Run _ (Either String GitResult) - pull repoKey = do + pull repoKey = withRepoLock env.process env.repoLocks repoKey do let path = repoPath repoKey address = repoAddress repoKey diff --git a/app/src/App/Main.purs b/app/src/App/Main.purs index e638cc68..efacbbf3 100644 --- a/app/src/App/Main.purs +++ b/app/src/App/Main.purs @@ -8,21 +8,24 @@ import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch.Retry as Fetch.Retry import Node.Process as Process -import Registry.App.Server.Env (ServerEnv, createServerEnv) +import Registry.App.Effect.Registry as Registry +import Registry.App.Server.Env (createServerEnv) import Registry.App.Server.JobExecutor as JobExecutor import Registry.App.Server.Router as Router +import Registry.App.Server.Scheduler as Scheduler main :: Effect Unit -main = do - createServerEnv # Aff.runAff_ case _ of - Left error -> do +main = Aff.launchAff_ do + Aff.attempt createServerEnv >>= case _ of + Left error -> liftEffect do Console.log $ "Failed to start server: " <> Aff.message error Process.exit' 1 - Right env -> do + Right env -> liftEffect do case env.vars.resourceEnv.healthchecksUrl of Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl - Aff.launchAff_ $ jobExecutor env + Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler (env { process = Registry.Scheduler }) + Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor (env { process = Registry.JobExecutor }) Router.runRouter env where healthcheck :: String -> Aff Unit @@ -63,20 +66,22 @@ main = do Succeeded _ -> do Console.error "Healthchecks returned non-200 status and failure limit reached, will not retry." - jobExecutor :: ServerEnv -> Aff Unit - jobExecutor env = do - loop initialRestartDelay + -- | Run an Aff action in a loop with exponential backoff on failure. + -- | If the action runs for longer than 60 seconds before failing, + -- | the restart delay resets to the initial value (heuristic for stability). + withRetryLoop :: String -> Aff (Either Aff.Error Unit) -> Aff Unit + withRetryLoop name action = loop initialRestartDelay where initialRestartDelay = Milliseconds 100.0 loop restartDelay = do start <- nowUTC - result <- JobExecutor.runJobExecutor env + result <- action end <- nowUTC Console.error case result of - Left error -> "Job executor failed: " <> Aff.message error - Right _ -> "Job executor exited for no reason." + Left error -> name <> " failed: " <> Aff.message error + Right _ -> name <> " exited for no reason." -- This is a heuristic: if the executor keeps crashing immediately, we -- restart with an exponentially increasing delay, but once the executor diff --git a/app/src/App/Server/Env.purs b/app/src/App/Server/Env.purs index 70e5698f..53b71d4d 100644 --- a/app/src/App/Server/Env.purs +++ b/app/src/App/Server/Env.purs @@ -28,7 +28,7 @@ import Registry.App.Effect.PackageSets (PACKAGE_SETS) import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Pursuit (PURSUIT) import Registry.App.Effect.Pursuit as Pursuit -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (Process, REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (SOURCE) import Registry.App.Effect.Source as Source @@ -76,6 +76,8 @@ type ServerEnv = , octokit :: Octokit , vars :: ServerEnvVars , debouncer :: Registry.Debouncer + , repoLocks :: Registry.RepoLocks + , process :: Process , db :: SQLite , jobId :: Maybe JobId } @@ -94,6 +96,7 @@ createServerEnv = do octokit <- Octokit.newOctokit vars.token vars.resourceEnv.githubApiUrl debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks db <- liftEffect $ SQLite.connect { database: vars.resourceEnv.databaseUrl.path @@ -111,6 +114,7 @@ createServerEnv = do pure { debouncer + , repoLocks , githubCacheRef , legacyCacheRef , registryCacheRef @@ -119,6 +123,7 @@ createServerEnv = do , vars , octokit , db + , process: Registry.API , jobId: Nothing } @@ -159,6 +164,8 @@ runEffects env operation = Aff.attempt do , workdir: scratchDir , debouncer: env.debouncer , cacheRef: env.registryCacheRef + , repoLocks: env.repoLocks + , process: env.process } ) # Archive.interpret Archive.handle diff --git a/app/src/App/Server/Scheduler.purs b/app/src/App/Server/Scheduler.purs new file mode 100644 index 00000000..1d1b2d5d --- /dev/null +++ b/app/src/App/Server/Scheduler.purs @@ -0,0 +1,305 @@ +module Registry.App.Server.Scheduler + ( runScheduler + ) where + +import Registry.App.Prelude + +import Data.Array as Array +import Data.Array.NonEmpty as NonEmptyArray +import Data.DateTime as DateTime +import Data.Map as Map +import Data.Set as Set +import Data.Set.NonEmpty (NonEmptySet) +import Data.Set.NonEmpty as NonEmptySet +import Data.String as String +import Data.Time.Duration (Hours(..)) +import Effect.Aff (Milliseconds(..)) +import Effect.Aff as Aff +import Registry.App.Auth as Auth +import Registry.App.CLI.PursVersions as PursVersions +import Registry.App.Effect.Db as Db +import Registry.App.Effect.Env as Env +import Registry.App.Effect.GitHub as GitHub +import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageSets as PackageSets +import Registry.App.Effect.Registry as Registry +import Registry.App.Legacy.LenientVersion as LenientVersion +import Registry.App.Server.Env (ServerEffects, ServerEnv, runEffects) +import Registry.Foreign.Octokit as Octokit +import Registry.Location (Location(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.PackageSet (PackageSet(..)) +import Registry.Range as Range +import Run (Run) + +-- | The scheduler loop runs immediately, then every 24 hours. +-- | It checks for work that needs to be enqueued (transfers, package set +-- | updates, legacy imports) and creates the appropriate jobs. +runScheduler :: ServerEnv -> Aff (Either Aff.Error Unit) +runScheduler env = runEffects env do + Log.info "Starting Scheduler" + loop + where + sleepTime = Milliseconds (1000.0 * 60.0 * 60.0 * 24.0) + + loop = do + -- Run all scheduling checks + scheduleTransfers + schedulePackageSetUpdates + scheduleDailyPublish + Log.info "Scheduler cycle complete, sleeping for 24 hours..." + -- Sleep for a while, then run again + liftAff $ Aff.delay sleepTime + loop + +-- | Check for packages that have moved and enqueue transfer jobs. +scheduleTransfers :: Run ServerEffects Unit +scheduleTransfers = do + Log.info "Scheduler: checking for package transfers..." + allMetadata <- Registry.readAllMetadata + + -- Check each package for location changes + transfersNeeded <- Array.catMaybes <$> for (Map.toUnfoldable allMetadata) \(Tuple name (Metadata metadata)) -> + case metadata.location of + Git _ -> pure Nothing -- Skip non-GitHub packages + GitHub registered -> do + -- Fetch tags to see if repo has moved + GitHub.listTags { owner: registered.owner, repo: registered.repo } >>= case _ of + Left _ -> pure Nothing -- Can't fetch tags, skip + Right tags | Array.null tags -> pure Nothing -- No tags, skip + Right tags -> case Array.head tags of + Nothing -> pure Nothing + Just tag -> + -- Parse the tag URL to get actual current location + case tagUrlToRepoUrl tag.url of + Nothing -> pure Nothing + Just actual + | locationsMatch registered actual -> pure Nothing -- No change + | otherwise -> pure $ Just { name, newLocation: GitHub { owner: actual.owner, repo: actual.repo, subdir: registered.subdir } } + + case Array.length transfersNeeded of + 0 -> Log.info "No packages require transferring." + n -> do + Log.info $ show n <> " packages need transferring" + for_ transfersNeeded \{ name, newLocation } -> + enqueueTransferJob name newLocation + +-- | Parse GitHub API tag URL to extract owner/repo +-- | Example: https://api.github.com/repos/octocat/Hello-World/commits/abc123 +tagUrlToRepoUrl :: String -> Maybe { owner :: String, repo :: String } +tagUrlToRepoUrl url = do + noPrefix <- String.stripPrefix (String.Pattern "https://api.github.com/repos/") url + case Array.take 2 $ String.split (String.Pattern "/") noPrefix of + [ owner, repo ] -> Just { owner, repo: String.toLower repo } + _ -> Nothing + +-- | Case-insensitive comparison of GitHub locations +locationsMatch :: forall r. { owner :: String, repo :: String | r } -> { owner :: String, repo :: String } -> Boolean +locationsMatch loc1 loc2 = + String.toLower loc1.owner == String.toLower loc2.owner + && String.toLower loc1.repo + == String.toLower loc2.repo + +enqueueTransferJob :: PackageName -> Location -> Run ServerEffects Unit +enqueueTransferJob name newLocation = do + -- Check if transfer job already exists + existingJob <- Db.selectTransferJob name + case existingJob of + Just _ -> Log.debug $ "Transfer job already exists for " <> PackageName.print name + Nothing -> do + let payload = { name, newLocation } + let rawPayload = stringifyJson Operation.transferCodec payload + { privateKey } <- Env.askPacchettiBotti + case Auth.signPayload { privateKey, rawPayload } of + Left _ -> Log.error $ "Failed to sign transfer for " <> PackageName.print name + Right signature -> do + jobId <- Db.insertTransferJob { payload, rawPayload, signature } + Log.info $ "Enqueued transfer job " <> unwrap jobId <> " for " <> PackageName.print name + +-- | Check for recent uploads and enqueue package set update job. +schedulePackageSetUpdates :: Run ServerEffects Unit +schedulePackageSetUpdates = do + Log.info "Scheduler: checking for package set updates..." + + -- Get the current package set + latestPackageSet <- Registry.readLatestPackageSet >>= case _ of + Nothing -> do + Log.warn "No package set found, skipping package set updates" + pure Nothing + Just set -> pure (Just set) + + for_ latestPackageSet \packageSet -> do + let currentPackages = (un PackageSet packageSet).packages + + -- Find packages uploaded in the last 24 hours that aren't already in the set + recentUploads <- findRecentUploads (Hours 24.0) + let + -- Filter out packages already in the set at the same or newer version + newOrUpdated = recentUploads # Map.filterWithKey \name version -> + case Map.lookup name currentPackages of + -- new package goes in + Nothing -> true + -- as do existing packages with a newer version + Just currentVersion -> version > currentVersion + + if Map.isEmpty newOrUpdated then + Log.info "No new packages for package set update." + else do + Log.info $ "Found " <> show (Map.size newOrUpdated) <> " candidates to validate" + + -- Pre-validate candidates to filter out packages with missing dependencies + manifestIndex <- Registry.readAllManifests + let candidates = PackageSets.validatePackageSetCandidates manifestIndex packageSet (map Just newOrUpdated) + + unless (Map.isEmpty candidates.rejected) do + Log.info $ "Some packages are not eligible for the package set:\n" <> PackageSets.printRejections candidates.rejected + + -- Only enqueue accepted packages (filter out removals, keep only updates) + let accepted = Map.catMaybes candidates.accepted + + if Map.isEmpty accepted then + Log.info "No packages passed validation for package set update." + else do + Log.info $ "Validated " <> show (Map.size accepted) <> " packages for package set update" + + -- Create a package set update payload with only validated packages + let + payload = Operation.PackageSetUpdate + { compiler: Nothing -- Use current compiler + , packages: map Just accepted -- Just version = add/update + } + rawPayload = stringifyJson Operation.packageSetOperationCodec payload + + -- Check if a similar job already exists + existingJob <- Db.selectPackageSetJobByPayload payload + case existingJob of + Just _ -> Log.debug "Package set job with same payload already exists" + Nothing -> do + -- No signature needed for package additions (only for compiler upgrades) + jobId <- Db.insertPackageSetJob { payload, rawPayload, signature: Nothing } + Log.info $ "Enqueued package set job " <> unwrap jobId + +-- | Find the latest version of each package uploaded within the time limit +findRecentUploads :: Hours -> Run ServerEffects (Map PackageName Version) +findRecentUploads limit = do + allMetadata <- Registry.readAllMetadata + now <- nowUTC + + let + getLatestRecentVersion :: Metadata -> Maybe Version + getLatestRecentVersion (Metadata metadata) = do + let + recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) + \(Tuple version { publishedTime }) -> if (DateTime.diff now publishedTime) <= limit then Just version else Nothing + Array.last $ Array.sort recentVersions + + pure $ Map.fromFoldable $ Array.catMaybes $ flip map (Map.toUnfoldable allMetadata) \(Tuple name metadata) -> + map (Tuple name) $ getLatestRecentVersion metadata + +-- | Check for new tags on existing packages and enqueue publish jobs for +-- | versions not yet published. This allows the registry to automatically +-- | publish new versions of packages that are already in the registry. +scheduleDailyPublish :: Run ServerEffects Unit +scheduleDailyPublish = do + Log.info "Scheduler: checking for new package versions..." + + allMetadata <- Registry.readAllMetadata + let packages = Map.toUnfoldable allMetadata :: Array (Tuple PackageName Metadata) + + for_ packages \(Tuple name (Metadata metadata)) -> do + case metadata.location of + Git _ -> pure unit -- Skip non-GitHub packages for now + GitHub { owner, repo } -> do + GitHub.listTags { owner, repo } >>= case _ of + Left err -> do + Log.debug $ "Failed to fetch tags for " <> PackageName.print name <> ": " <> Octokit.printGitHubError err + Right tags -> do + let + -- Combine published and unpublished versions into a set + publishedVersions = Set.fromFoldable + $ Map.keys metadata.published + <> Map.keys metadata.unpublished + + -- Parse tags as versions and filter out already published ones + newVersions = Array.catMaybes $ tags <#> \tag -> + case LenientVersion.parse tag.name of + Left _ -> Nothing -- Not a valid version tag + Right result -> + let + version = LenientVersion.version result + in + if Set.member version publishedVersions then Nothing -- Already published + else Just { version, ref: tag.name } + + for_ newVersions \{ version, ref } -> + enqueuePublishJob allMetadata name (Metadata metadata) version ref + +-- | Enqueue a publish job for a new package version discovered by the scheduler. +-- | Attempts to find a compatible compiler by looking at the previous version's +-- | dependencies. Falls back to the lowest compiler from the previous version if +-- | no dependencies exist, or to the latest compiler if no previous version exists. +enqueuePublishJob :: Map PackageName Metadata -> PackageName -> Metadata -> Version -> String -> Run ServerEffects Unit +enqueuePublishJob allMetadata name (Metadata metadata) version ref = do + -- Check if a publish job already exists for this package version + existingJob <- Db.selectPublishJob name version + case existingJob of + Just _ -> Log.debug $ "Publish job already exists for " <> formatPackageVersion name version + Nothing -> do + -- Try to find a compatible compiler by looking at the previous version's dependencies + compiler <- case Map.findMax metadata.published of + Just { key: prevVersion, value: publishedInfo } -> do + -- Look up the manifest for the previous version to get its dependencies + maybeManifest <- Registry.readManifest name prevVersion + case maybeManifest of + Just (Manifest manifest) | not (Map.isEmpty manifest.dependencies) -> do + -- Use previous version's dependencies to find compatible compilers + -- Find the highest published version of each dependency within its range + let + depVersions :: Map PackageName Version + depVersions = Map.mapMaybeWithKey (\depName range -> + case Map.lookup depName allMetadata of + Just (Metadata depMeta) -> + Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published + Nothing -> Nothing + ) manifest.dependencies + + case compatibleCompilers allMetadata depVersions of + Just compilerSet -> pure $ NonEmptySet.min compilerSet + -- No intersection found, fall back to lowest compiler from previous version + Nothing -> pure $ NonEmptyArray.head publishedInfo.compilers + -- No manifest or no dependencies, fall back to lowest compiler from previous version + _ -> pure $ NonEmptyArray.head publishedInfo.compilers + Nothing -> + NonEmptyArray.last <$> PursVersions.pursVersions + let + payload = + { name + -- Don't specify location - use current metadata location at publish time. + -- This avoids race conditions with transfer jobs that may update the location. + , location: Nothing + , ref + , version + , compiler + , resolutions: Nothing + } + jobId <- Db.insertPublishJob { payload } + Log.info $ "Enqueued publish job " <> unwrap jobId <> " for " <> formatPackageVersion name version + +-- | Given a set of package versions, determine the set of compilers that can be +-- | used for all packages by intersecting their supported compiler ranges. +compatibleCompilers :: Map PackageName Metadata -> Map PackageName Version -> Maybe (NonEmptySet Version) +compatibleCompilers allMetadata resolutions = do + let + associated :: Array { compilers :: NonEmptyArray Version } + associated = Map.toUnfoldableUnordered resolutions # Array.mapMaybe \(Tuple depName depVersion) -> do + Metadata depMeta <- Map.lookup depName allMetadata + published <- Map.lookup depVersion depMeta.published + Just { compilers: published.compilers } + + case Array.uncons associated of + Nothing -> Nothing + Just { head, tail: [] } -> Just $ NonEmptySet.fromFoldable1 head.compilers + Just { head, tail } -> do + let foldFn prev = Set.intersection prev <<< Set.fromFoldable <<< _.compilers + NonEmptySet.fromFoldable $ Array.foldl foldFn (Set.fromFoldable head.compilers) tail diff --git a/db/schema.sql b/db/schema.sql index 65319293..17e06c8d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -40,6 +40,8 @@ CREATE TABLE matrix_jobs ( CREATE TABLE package_set_jobs ( jobId TEXT PRIMARY KEY NOT NULL, payload JSON NOT NULL, + rawPayload TEXT NOT NULL, + signature TEXT, FOREIGN KEY (jobId) REFERENCES job_info (jobId) ON DELETE CASCADE ); CREATE TABLE logs ( diff --git a/nix/overlay.nix b/nix/overlay.nix index 8ec743a3..8858c140 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -62,14 +62,6 @@ let module = "Registry.Scripts.PackageDeleter"; description = "Delete packages from the registry"; }; - package-set-updater = { - module = "Registry.Scripts.PackageSetUpdater"; - description = "Update package sets"; - }; - package-transferrer = { - module = "Registry.Scripts.PackageTransferrer"; - description = "Transfer packages between storage backends"; - }; solver = { module = "Registry.Scripts.Solver"; description = "Run dependency solver against registry manifests"; diff --git a/nix/test/config.nix b/nix/test/config.nix index 07917444..afd6f187 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -172,6 +172,55 @@ let }; }; + # Unsafe-coerce package helpers (unsafe-coerce@6.0.0) + unsafeCoerceBase64Response = + fileName: + base64Response { + url = "/repos/purescript/purescript-unsafe-coerce/contents/${fileName}?ref=v6.0.0"; + inherit fileName; + filePath = rootPath + "/app/fixtures/github-packages/unsafe-coerce-6.0.0/${fileName}"; + }; + + unsafeCoerce404Response = fileName: { + request = { + method = "GET"; + url = "/repos/purescript/purescript-unsafe-coerce/contents/${fileName}?ref=v6.0.0"; + }; + response = { + status = 404; + headers."Content-Type" = "application/json"; + jsonBody = { + message = "Not Found"; + documentation_url = "https://docs.github.com/rest/repos/contents#get-repository-content"; + }; + }; + }; + + # Type-equality package helpers (type-equality@4.0.2) + # Note: Uses purescript owner (actual location) not old-owner (metadata location) + typeEqualityBase64Response = + fileName: + base64Response { + url = "/repos/purescript/purescript-type-equality/contents/${fileName}?ref=v4.0.2"; + inherit fileName; + filePath = rootPath + "/app/fixtures/github-packages/type-equality-4.0.1/${fileName}"; + }; + + typeEquality404Response = fileName: { + request = { + method = "GET"; + url = "/repos/purescript/purescript-type-equality/contents/${fileName}?ref=v4.0.2"; + }; + response = { + status = 404; + headers."Content-Type" = "application/json"; + jsonBody = { + message = "Not Found"; + documentation_url = "https://docs.github.com/rest/repos/contents#get-repository-content"; + }; + }; + }; + # GitHub API wiremock mappings githubMappings = [ (effectBase64Response "bower.json") @@ -188,6 +237,20 @@ let (console404Response "spago.dhall") (console404Response "purs.json") (console404Response "package.json") + # Unsafe-coerce package (unsafe-coerce@6.0.0) + (unsafeCoerceBase64Response "bower.json") + (unsafeCoerce404Response "LICENSE") + (unsafeCoerce404Response "spago.yaml") + (unsafeCoerce404Response "spago.dhall") + (unsafeCoerce404Response "purs.json") + (unsafeCoerce404Response "package.json") + # Type-equality package (type-equality@4.0.2 for legacy imports test) + (typeEqualityBase64Response "bower.json") + (typeEqualityBase64Response "LICENSE") + (typeEquality404Response "spago.yaml") + (typeEquality404Response "spago.dhall") + (typeEquality404Response "purs.json") + (typeEquality404Response "package.json") { request = { method = "GET"; @@ -205,6 +268,57 @@ let }; }; } + # Tags for prelude package (only v6.0.1 which is already published) + { + request = { + method = "GET"; + url = "/repos/purescript/purescript-prelude/tags"; + }; + response = { + status = 200; + headers."Content-Type" = "application/json"; + jsonBody = [ + { + name = "v6.0.1"; + commit = { + sha = "abc123def456"; + url = "https://api.github.com/repos/purescript/purescript-prelude/commits/abc123def456"; + }; + } + ]; + }; + } + # Tags for type-equality package (used by two scheduler tests): + # 1. Transfer detection: metadata says old-owner, commit URLs point to purescript + # 2. Legacy imports: v4.0.2 is a new version not yet published + { + request = { + method = "GET"; + url = "/repos/old-owner/purescript-type-equality/tags"; + }; + response = { + status = 200; + headers."Content-Type" = "application/json"; + jsonBody = [ + { + name = "v4.0.1"; + commit = { + sha = "type-eq-sha-401"; + # Points to actual owner - scheduler detects this transfer + url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-401"; + }; + } + { + name = "v4.0.2"; + commit = { + sha = "type-eq-sha-402"; + # New version not yet published - scheduler detects for legacy import + url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-402"; + }; + } + ]; + }; + } # Accept issue comment creation (used by GitHubIssue workflow) { request = { @@ -281,10 +395,20 @@ let ) ); - # Metadata fixtures directory (to determine which packages are "published") + # Metadata fixtures directory (to determine which package versions are "published") metadataFixturesDir = rootPath + "/app/fixtures/registry/metadata"; metadataFiles = builtins.attrNames (builtins.readDir metadataFixturesDir); - publishedPackageNames = map (f: lib.removeSuffix ".json" f) metadataFiles; + + # Parse metadata files to get the actual published versions (not just package names) + # Returns a set like { "prelude-6.0.1" = true; "type-equality-4.0.1" = true; } + publishedVersions = lib.foldl' (acc: fileName: + let + packageName = lib.removeSuffix ".json" fileName; + metadata = builtins.fromJSON (builtins.readFile (metadataFixturesDir + "/${fileName}")); + versions = builtins.attrNames (metadata.published or {}); + in + acc // lib.genAttrs (map (v: "${packageName}-${v}") versions) (_: true) + ) {} metadataFiles; # ============================================================================ # UNIFIED STORAGE MAPPINGS WITH WIREMOCK SCENARIOS @@ -298,9 +422,9 @@ let # Scenario design: # - One scenario per package-version (e.g., "effect-4.0.0") # - WireMock scenarios always start at state "Started" - # - Published packages (has metadata): "Started" means Present (tarball available) + # - Published versions (version exists in metadata.published): "Started" means Present # - After DELETE, transitions to "Deleted" state (404 on GET) - # - Unpublished packages (no metadata): "Started" means Absent (tarball 404) + # - Unpublished versions (new version not in metadata): "Started" means Absent (404) # - After PUT upload, transitions to "Present" state # - After DELETE, transitions to "Deleted" state (404 on GET) # @@ -316,7 +440,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; tarPath = "/${pkg.name}/${pkg.version}.tar.gz"; in if isPublished then @@ -407,7 +531,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; escapedName = lib.replaceStrings [ "-" ] [ "\\-" ] pkg.name; listUrlPattern = "/\\?.*prefix=${escapedName}.*"; presentContents = ''${pkg.name}/${pkg.version}.tar.gz1000"abc123"''; @@ -492,7 +616,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; escapedVersion = lib.replaceStrings [ "." ] [ "\\." ] pkg.version; urlPattern = "/${pkg.name}/${escapedVersion}\\.tar\\.gz.*"; in @@ -618,7 +742,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; versionsUrl = "/packages/purescript-${pkg.name}/available-versions"; publishedVersionsBody = ''[["${pkg.version}","https://pursuit.purescript.org/packages/purescript-${pkg.name}/${pkg.version}"]]''; in @@ -781,7 +905,10 @@ let # Script to set up git fixtures setupGitFixtures = pkgs.writeShellApplication { name = "setup-git-fixtures"; - runtimeInputs = [ pkgs.git ]; + runtimeInputs = [ + pkgs.git + pkgs.jq + ]; text = '' FIXTURES_DIR="''${1:-${stateDir}/repo-fixtures}" @@ -800,8 +927,19 @@ let cp -r ${rootPath}/app/fixtures/{registry-index,registry,package-sets} "$FIXTURES_DIR/purescript/" cp -r ${rootPath}/app/fixtures/github-packages/effect-4.0.0 "$FIXTURES_DIR/purescript/purescript-effect" cp -r ${rootPath}/app/fixtures/github-packages/console-6.1.0 "$FIXTURES_DIR/purescript/purescript-console" + cp -r ${rootPath}/app/fixtures/github-packages/unsafe-coerce-6.0.0 "$FIXTURES_DIR/purescript/purescript-unsafe-coerce" + cp -r ${rootPath}/app/fixtures/github-packages/type-equality-4.0.1 "$FIXTURES_DIR/purescript/purescript-type-equality" chmod -R u+w "$FIXTURES_DIR/purescript" + # Set type-equality publishedTime to current time for package set update test + # This makes type-equality appear as a "recent upload" so the scheduler will + # detect it and enqueue a package set update job + current_time=$(date -u +"%Y-%m-%dT%H:%M:%S.000Z") + jq --arg time "$current_time" \ + '.published["4.0.1"].publishedTime = $time' \ + "$FIXTURES_DIR/purescript/registry/metadata/type-equality.json" > temp.json && \ + mv temp.json "$FIXTURES_DIR/purescript/registry/metadata/type-equality.json" + for repo in "$FIXTURES_DIR"/purescript/*/; do cd "$repo" git init -b master && git add . @@ -814,6 +952,9 @@ let gitbot -C "$FIXTURES_DIR/purescript/package-sets" tag -m "psc-0.15.9-20230105" psc-0.15.9-20230105 gitbot -C "$FIXTURES_DIR/purescript/purescript-effect" tag -m "v4.0.0" v4.0.0 gitbot -C "$FIXTURES_DIR/purescript/purescript-console" tag -m "v6.1.0" v6.1.0 + gitbot -C "$FIXTURES_DIR/purescript/purescript-unsafe-coerce" tag -m "v6.0.0" v6.0.0 + gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" tag -m "v4.0.1" v4.0.1 + gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" tag -m "v4.0.2" v4.0.2 ''; }; diff --git a/scripts/src/ArchiveSeeder.purs b/scripts/src/ArchiveSeeder.purs index fe0ae805..ca4fdcdd 100644 --- a/scripts/src/ArchiveSeeder.purs +++ b/scripts/src/ArchiveSeeder.purs @@ -95,7 +95,18 @@ main = launchAff_ do runAppEffects <- do debouncer <- Registry.newDebouncer - let registryEnv = { pull: Git.Autostash, write: Registry.ReadOnly, repos: Registry.defaultRepos, workdir: scratchDir, debouncer, cacheRef: registryCacheRef } + repoLocks <- Registry.newRepoLocks + let + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptArchiveSeeder + } token <- Env.lookupRequired Env.githubToken s3 <- lift2 { key: _, secret: _ } (Env.lookupRequired Env.spacesKey) (Env.lookupRequired Env.spacesSecret) diff --git a/scripts/src/CompilerVersions.purs b/scripts/src/CompilerVersions.purs index 127d0b97..176a200a 100644 --- a/scripts/src/CompilerVersions.purs +++ b/scripts/src/CompilerVersions.purs @@ -119,6 +119,7 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -128,6 +129,8 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptCompilerVersions } -- Logging diff --git a/scripts/src/LegacyImporter.purs b/scripts/src/LegacyImporter.purs index 05e73ae8..2decb34b 100644 --- a/scripts/src/LegacyImporter.purs +++ b/scripts/src/LegacyImporter.purs @@ -189,7 +189,18 @@ main = launchAff_ do -- uploaded and manifests and metadata are written, committed, and pushed. runAppEffects <- do debouncer <- Registry.newDebouncer - let registryEnv pull write = { pull, write, repos: Registry.defaultRepos, workdir: scratchDir, debouncer, cacheRef: registryCacheRef } + repoLocks <- Registry.newRepoLocks + let + registryEnv pull write = + { pull + , write + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptLegacyImporter + } case mode of DryRun -> do token <- Env.lookupRequired Env.githubToken diff --git a/scripts/src/PackageDeleter.purs b/scripts/src/PackageDeleter.purs index 257a7b1a..d93d268b 100644 --- a/scripts/src/PackageDeleter.purs +++ b/scripts/src/PackageDeleter.purs @@ -122,6 +122,7 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -131,6 +132,8 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptPackageDeleter } -- Logging diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs deleted file mode 100644 index 29423cf7..00000000 --- a/scripts/src/PackageSetUpdater.purs +++ /dev/null @@ -1,192 +0,0 @@ -module Registry.Scripts.PackageSetUpdater where - -import Registry.App.Prelude - -import ArgParse.Basic (ArgParser) -import ArgParse.Basic as Arg -import Data.Array as Array -import Data.Array.NonEmpty as NonEmptyArray -import Data.DateTime as DateTime -import Data.FoldableWithIndex (foldMapWithIndex) -import Data.Formatter.DateTime as Formatter.DateTime -import Data.Map as Map -import Data.Number.Format as Number.Format -import Data.String as String -import Data.Time.Duration (Hours(..)) -import Effect.Aff as Aff -import Effect.Class.Console as Console -import Node.Path as Path -import Node.Process as Process -import Registry.App.CLI.Git as Git -import Registry.App.Effect.Cache as Cache -import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub as GitHub -import Registry.App.Effect.Log (LOG) -import Registry.App.Effect.Log as Log -import Registry.App.Effect.PackageSets (Change(..), PACKAGE_SETS) -import Registry.App.Effect.PackageSets as PackageSets -import Registry.App.Effect.Registry (REGISTRY) -import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage as Storage -import Registry.Foreign.FSExtra as FS.Extra -import Registry.Foreign.Octokit as Octokit -import Registry.Internal.Format as Internal.Format -import Registry.Version as Version -import Run (AFF, EFFECT, Run) -import Run as Run -import Run.Except (EXCEPT) -import Run.Except as Except - -data PublishMode = GeneratePackageSet | CommitPackageSet - -derive instance Eq PublishMode - -parser :: ArgParser PublishMode -parser = Arg.choose "command" - [ Arg.flag [ "generate" ] - "Generate a new package set without committing the results." - $> GeneratePackageSet - , Arg.flag [ "commit" ] - "Generate a new package set and commit the results." - $> CommitPackageSet - ] - -main :: Effect Unit -main = Aff.launchAff_ do - args <- Array.drop 2 <$> liftEffect Process.argv - let description = "A script for updating the package sets." - mode <- case Arg.parseArgs "package-set-updater" description parser args of - Left err -> Console.log (Arg.printArgError err) *> liftEffect (Process.exit' 1) - Right command -> pure command - - -- Environment - _ <- Env.loadEnvFile ".env" - - { token, write } <- case mode of - GeneratePackageSet -> do - Env.lookupOptional Env.githubToken >>= case _ of - Nothing -> do - token <- Env.lookupRequired Env.pacchettibottiToken - pure { token, write: Registry.ReadOnly } - Just token -> - pure { token, write: Registry.ReadOnly } - CommitPackageSet -> do - token <- Env.lookupRequired Env.pacchettibottiToken - pure { token, write: Registry.CommitAs (Git.pacchettibottiCommitter token) } - - -- Package sets - let packageSetsEnv = { workdir: Path.concat [ scratchDir, "package-set-build" ] } - - -- GitHub - resourceEnv <- Env.lookupResourceEnv - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl - - -- Caching - let cache = Path.concat [ scratchDir, ".cache" ] - FS.Extra.ensureDirectory cache - githubCacheRef <- Cache.newCacheRef - registryCacheRef <- Cache.newCacheRef - - -- Registry - debouncer <- Registry.newDebouncer - let - registryEnv :: Registry.RegistryEnv - registryEnv = - { write - , pull: Git.ForceClean - , repos: Registry.defaultRepos - , workdir: scratchDir - , debouncer - , cacheRef: registryCacheRef - } - - -- Logging - now <- nowUTC - let logDir = Path.concat [ scratchDir, "logs" ] - FS.Extra.ensureDirectory logDir - let logFile = "package-set-updater-" <> String.take 19 (Formatter.DateTime.format Internal.Format.iso8601DateTime now) <> ".log" - let logPath = Path.concat [ logDir, logFile ] - - updater - # PackageSets.interpret (PackageSets.handle packageSetsEnv) - # Registry.interpret (Registry.handle registryEnv) - # Storage.interpret (Storage.handleReadOnly cache) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Except.catch (\msg -> Log.error msg *> Run.liftEffect (Process.exit' 1)) - # Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' - -updater :: forall r. Run (REGISTRY + PACKAGE_SETS + LOG + EXCEPT String + AFF + EFFECT + r) Unit -updater = do - prevPackageSet <- Registry.readLatestPackageSet >>= case _ of - Nothing -> Except.throw "No previous package set found, cannot continue." - Just set -> pure set - - PackageSets.validatePackageSet prevPackageSet - - let compiler = (un PackageSet prevPackageSet).compiler - - Log.info $ "Using compiler " <> Version.print compiler - - let uploadHours = 24.0 - recentUploads <- findRecentUploads (Hours uploadHours) - - manifestIndex <- Registry.readAllManifests - let candidates = PackageSets.validatePackageSetCandidates manifestIndex prevPackageSet (map Just recentUploads.eligible) - unless (Map.isEmpty candidates.rejected) do - Log.info $ "Some packages uploaded in the last " <> Number.Format.toString uploadHours <> " hours are not eligible for the automated package sets." - Log.info $ PackageSets.printRejections candidates.rejected - - if Map.isEmpty candidates.accepted then do - Log.info "No eligible additions, updates, or removals to produce a new package set." - else do - -- You can't remove packages via the automatic updater. - let eligible = Map.catMaybes candidates.accepted - let listPackages = foldMapWithIndex \name version -> [ formatPackageVersion name version ] - Log.info $ "Found package versions eligible for inclusion in package set: " <> Array.foldMap (append "\n - ") (listPackages eligible) - PackageSets.upgradeSequential prevPackageSet compiler (map (maybe Remove Update) candidates.accepted) >>= case _ of - Nothing -> do - Log.info "No packages could be added to the set. All packages failed." - Just { failed, succeeded, result } -> do - let - listChanges = foldMapWithIndex \name -> case _ of - Remove -> [] - Update version -> [ formatPackageVersion name version ] - unless (Map.isEmpty failed) do - Log.info $ "Some packages could not be added to the set: " <> Array.foldMap (append "\n - ") (listChanges failed) - Log.info $ "New packages were added to the set: " <> Array.foldMap (append "\n - ") (listChanges succeeded) - -- We only include the successful changes in the commit message. - let commitMessage = PackageSets.commitMessage prevPackageSet succeeded (un PackageSet result).version - Registry.writePackageSet result commitMessage - Log.info "Built and released a new package set! Now mirroring to the package-sets repo..." - Registry.mirrorPackageSet result - Log.info "Mirrored a new legacy package set." - -type RecentUploads = - { eligible :: Map PackageName Version - , ineligible :: Map PackageName (NonEmptyArray Version) - } - -findRecentUploads :: forall r. Hours -> Run (REGISTRY + EXCEPT String + EFFECT + r) RecentUploads -findRecentUploads limit = do - allMetadata <- Registry.readAllMetadata - now <- nowUTC - - let - uploads = Map.fromFoldable do - Tuple name (Metadata metadata) <- Map.toUnfoldable allMetadata - versions <- Array.fromFoldable $ NonEmptyArray.fromArray do - Tuple version { publishedTime } <- Map.toUnfoldable metadata.published - let diff = DateTime.diff now publishedTime - guard (diff <= limit) - pure version - pure (Tuple name versions) - - deduplicated = uploads # flip foldlWithIndex { ineligible: Map.empty, eligible: Map.empty } \name acc versions -> do - let { init, last } = NonEmptyArray.unsnoc versions - case NonEmptyArray.fromArray init of - Nothing -> acc { eligible = Map.insert name last acc.eligible } - Just entries -> acc { eligible = Map.insert name last acc.eligible, ineligible = Map.insert name entries acc.ineligible } - - pure deduplicated diff --git a/scripts/src/PackageTransferrer.purs b/scripts/src/PackageTransferrer.purs deleted file mode 100644 index 31e85919..00000000 --- a/scripts/src/PackageTransferrer.purs +++ /dev/null @@ -1,215 +0,0 @@ -module Registry.Scripts.PackageTransferrer where - -import Registry.App.Prelude - -import Data.Array as Array -import Data.Codec.JSON as CJ -import Data.Codec.JSON.Common as CJ.Common -import Data.Codec.JSON.Record as CJ.Record -import Data.Formatter.DateTime as Formatter.DateTime -import Data.Map as Map -import Data.String as String -import Effect.Ref as Ref -import Node.Path as Path -import Node.Process as Process -import Registry.App.API as API -import Registry.App.Auth as Auth -import Registry.App.CLI.Git as Git -import Registry.App.Effect.Cache as Cache -import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub (GITHUB) -import Registry.App.Effect.GitHub as GitHub -import Registry.App.Effect.Log (LOG) -import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) -import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage as Storage -import Registry.App.Legacy.LenientVersion as LenientVersion -import Registry.App.Legacy.Types (RawPackageName(..)) -import Registry.Foreign.FSExtra as FS.Extra -import Registry.Foreign.Octokit (Tag) -import Registry.Foreign.Octokit as Octokit -import Registry.Internal.Format as Internal.Format -import Registry.Location as Location -import Registry.Operation (AuthenticatedPackageOperation(..)) -import Registry.Operation as Operation -import Registry.Operation.Validation as Operation.Validation -import Registry.PackageName as PackageName -import Registry.Scripts.LegacyImporter as LegacyImporter -import Run (Run) -import Run as Run -import Run.Except (EXCEPT) -import Run.Except as Except -import Run.Except as Run.Except - -main :: Effect Unit -main = launchAff_ do - - -- Environment - _ <- Env.loadEnvFile ".env" - token <- Env.lookupRequired Env.pacchettibottiToken - publicKey <- Env.lookupRequired Env.pacchettibottiED25519Pub - privateKey <- Env.lookupRequired Env.pacchettibottiED25519 - resourceEnv <- Env.lookupResourceEnv - - -- Caching - let cache = Path.concat [ scratchDir, ".cache" ] - FS.Extra.ensureDirectory cache - githubCacheRef <- Cache.newCacheRef - registryCacheRef <- Cache.newCacheRef - - -- GitHub - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl - - -- Registry - debouncer <- Registry.newDebouncer - let - registryEnv :: Registry.RegistryEnv - registryEnv = - { write: Registry.CommitAs (Git.pacchettibottiCommitter token) - , pull: Git.ForceClean - , repos: Registry.defaultRepos - , workdir: scratchDir - , debouncer - , cacheRef: registryCacheRef - } - - -- Logging - now <- nowUTC - let logDir = Path.concat [ scratchDir, "logs" ] - FS.Extra.ensureDirectory logDir - let logFile = "package-transferrer-" <> String.take 19 (Formatter.DateTime.format Internal.Format.iso8601DateTime now) <> ".log" - let logPath = Path.concat [ logDir, logFile ] - - transfer - # Registry.interpret (Registry.handle registryEnv) - # Storage.interpret (Storage.handleReadOnly cache) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Except.catch (\msg -> Log.error msg *> Run.liftEffect (Process.exit' 1)) - # Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) - # Env.runPacchettiBottiEnv { privateKey, publicKey } - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' - -transfer :: forall r. Run (API.AuthenticatedEffects + r) Unit -transfer = do - Log.info "Processing legacy registry..." - allMetadata <- Registry.readAllMetadata - { bower, new } <- Registry.readLegacyRegistry - let packages = Map.union bower new - Log.info "Reading latest locations for legacy registry packages..." - locations <- latestLocations allMetadata packages - let needsTransfer = Map.catMaybes locations - case Map.size needsTransfer of - 0 -> Log.info "No packages require transferring." - n -> do - Log.info $ Array.fold [ show n, " packages need transferring: ", printJson (CJ.Common.strMap packageLocationsCodec) needsTransfer ] - _ <- transferAll packages needsTransfer - Log.info "Completed transfers!" - -transferAll :: forall r. Map String String -> Map String PackageLocations -> Run (API.AuthenticatedEffects + r) (Map String String) -transferAll packages packageLocations = do - packagesRef <- liftEffect (Ref.new packages) - forWithIndex_ packageLocations \package locations -> do - let newPackageLocation = locations.tagLocation - transferPackage package newPackageLocation - let url = locationToPackageUrl newPackageLocation - liftEffect $ Ref.modify_ (Map.insert package url) packagesRef - liftEffect $ Ref.read packagesRef - -transferPackage :: forall r. String -> Location -> Run (API.AuthenticatedEffects + r) Unit -transferPackage rawPackageName newLocation = do - name <- case PackageName.parse (stripPureScriptPrefix rawPackageName) of - Left _ -> Except.throw $ "Could not transfer " <> rawPackageName <> " because it is not a valid package name." - Right value -> pure value - - let - payload = { name, newLocation } - rawPayload = stringifyJson Operation.transferCodec payload - - { privateKey } <- Env.askPacchettiBotti - - signature <- case Auth.signPayload { privateKey, rawPayload } of - Left _ -> Except.throw "Error signing transfer." - Right signature -> pure signature - - API.authenticated - { payload: Transfer payload - , rawPayload - , signature - } - -type PackageLocations = - { registeredLocation :: Location - , tagLocation :: Location - } - -packageLocationsCodec :: CJ.Codec PackageLocations -packageLocationsCodec = CJ.named "PackageLocations" $ CJ.Record.object - { registeredLocation: Location.codec - , tagLocation: Location.codec - } - -latestLocations :: forall r. Map PackageName Metadata -> Map String String -> Run (REGISTRY + GITHUB + LOG + EXCEPT String + r) (Map String (Maybe PackageLocations)) -latestLocations allMetadata packages = forWithIndex packages \package location -> do - let rawName = RawPackageName (stripPureScriptPrefix package) - Run.Except.runExceptAt LegacyImporter._exceptPackage (LegacyImporter.validatePackage rawName location) >>= case _ of - Left { error: LegacyImporter.PackageURLRedirects { received, registered } } -> do - let newLocation = GitHub { owner: received.owner, repo: received.repo, subdir: Nothing } - Log.info $ "Package " <> package <> " has moved to " <> locationToPackageUrl newLocation - if Operation.Validation.locationIsUnique newLocation allMetadata then do - Log.info "New location is unique; package will be transferred." - pure $ Just - { registeredLocation: GitHub { owner: registered.owner, repo: registered.repo, subdir: Nothing } - , tagLocation: newLocation - } - else do - Log.info "Package will not be transferred! New location is already in use." - pure Nothing - Left _ -> pure Nothing - Right packageResult | Array.null packageResult.tags -> pure Nothing - Right packageResult -> do - Registry.readMetadata packageResult.name >>= case _ of - Nothing -> do - Log.error $ "Cannot verify location of " <> PackageName.print packageResult.name <> " because it has no metadata." - pure Nothing - Just metadata -> case latestPackageLocations packageResult metadata of - Left error -> do - Log.warn $ "Could not verify location of " <> PackageName.print packageResult.name <> ": " <> error - pure Nothing - Right locations - | locationsMatch locations.registeredLocation locations.tagLocation -> pure Nothing - | otherwise -> pure $ Just locations - where - -- The eq instance for locations has case sensitivity, but GitHub doesn't care. - locationsMatch :: Location -> Location -> Boolean - locationsMatch (GitHub location1) (GitHub location2) = - (String.toLower location1.repo == String.toLower location2.repo) - && (String.toLower location1.owner == String.toLower location2.owner) - locationsMatch _ _ = - unsafeCrashWith "Only GitHub locations can be considered in legacy registries." - -latestPackageLocations :: LegacyImporter.PackageResult -> Metadata -> Either String PackageLocations -latestPackageLocations package (Metadata { location, published }) = do - let - isMatchingTag :: Version -> Tag -> Boolean - isMatchingTag version tag = fromMaybe false do - tagVersion <- hush $ LenientVersion.parse tag.name - pure $ version == LenientVersion.version tagVersion - - matchingTag <- do - if Map.isEmpty published then do - note "No repo tags exist" $ Array.head package.tags - else do - Tuple version _ <- note "No published versions" $ Array.last (Map.toUnfoldable published) - note "No versions match repo tags" $ Array.find (isMatchingTag version) package.tags - tagUrl <- note ("Could not parse tag url " <> matchingTag.url) $ LegacyImporter.tagUrlToRepoUrl matchingTag.url - let tagLocation = GitHub { owner: tagUrl.owner, repo: tagUrl.repo, subdir: Nothing } - pure { registeredLocation: location, tagLocation } - -locationToPackageUrl :: Location -> String -locationToPackageUrl = case _ of - GitHub { owner, repo } -> - Array.fold [ "https://github.com/", owner, "/", repo, ".git" ] - Git _ -> - unsafeCrashWith "Git urls cannot be registered." diff --git a/scripts/src/Solver.purs b/scripts/src/Solver.purs index ce615b5a..a61bcacf 100644 --- a/scripts/src/Solver.purs +++ b/scripts/src/Solver.purs @@ -117,7 +117,18 @@ main = launchAff_ do FS.Extra.ensureDirectory cache debouncer <- Registry.newDebouncer - let registryEnv pull write = { pull, write, repos: Registry.defaultRepos, workdir: scratchDir, debouncer, cacheRef: registryCacheRef } + repoLocks <- Registry.newRepoLocks + let + registryEnv pull write = + { pull + , write + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptSolver + } resourceEnv <- Env.lookupResourceEnv token <- Env.lookupRequired Env.githubToken octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl diff --git a/scripts/src/VerifyIntegrity.purs b/scripts/src/VerifyIntegrity.purs index 97aef379..ac79bd7e 100644 --- a/scripts/src/VerifyIntegrity.purs +++ b/scripts/src/VerifyIntegrity.purs @@ -79,6 +79,7 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -88,6 +89,8 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptVerifyIntegrity } -- Logging