Rework daily cron jobs to hit the API instead of working directly on the Registry

app-e2e/spago.yaml

@@ -21,8 +21,10 @@ package:
     - registry-app
     - registry-foreign
     - registry-lib
+    - registry-scripts
     - registry-test-utils
     - routing-duplex
+    - run
     - spec
     - spec-node
     - strings

app-e2e/src/Test/E2E/Endpoint/PackageSets.purs

@@ -16,6 +16,10 @@ spec :: E2ESpec
 spec = do
   Spec.describe "Package Sets endpoint" do
     Spec.it "accepts unauthenticated add/upgrade requests" do
+      -- First publish unsafe-coerce to create the tarball in storage
+      { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData
+      _ <- Env.pollJobOrFail publishJobId
+      -- Now add it to the package set
       { jobId } <- Client.packageSets Fixtures.packageSetAddRequest
       job <- Env.pollJobOrFail jobId
       Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
@@ -47,6 +51,10 @@ spec = do
           Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
 
     Spec.it "returns existing job for duplicate requests" do
+      -- First publish unsafe-coerce so the package set request is valid
+      { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData
+      _ <- Env.pollJobOrFail publishJobId
+      -- Now test that duplicate requests return the same job ID
       { jobId: firstJobId } <- Client.packageSets Fixtures.packageSetAddRequest
       { jobId: secondJobId } <- Client.packageSets Fixtures.packageSetAddRequest
       Assert.shouldEqual firstJobId secondJobId

app-e2e/src/Test/E2E/Endpoint/Publish.purs

@@ -59,7 +59,7 @@ spec = do
           )
           allJobs
         -- The expected compilers are: the publish compiler + all matrix job compilers
-        expectedCompilers = Set.fromFoldable $ Array.cons Fixtures.effectPublishData.compiler matrixCompilers
+        expectedCompilers = Set.fromFoldable $ maybe matrixCompilers (\c -> Array.cons c matrixCompilers) Fixtures.effectPublishData.compiler
 
       Metadata metadataAfter <- Env.readMetadata Fixtures.effect.name
       case Map.lookup Fixtures.effect.version metadataAfter.published of

app-e2e/src/Test/E2E/Endpoint/Startup.purs

@@ -0,0 +1,49 @@
+-- | E2E tests for server startup behavior.
+-- |
+-- | IMPORTANT: These tests must run BEFORE resetTestState is called, since
+-- | the jobs are created at server startup and would be cleared.
+module Test.E2E.Endpoint.Startup (spec) where
+
+import Registry.App.Prelude
+
+import Data.Array as Array
+import Data.String as String
+import Registry.API.V1 (Job(..))
+import Registry.PackageName as PackageName
+import Registry.Test.Assert as Assert
+import Registry.Test.Utils as Utils
+import Test.E2E.Support.Client as Client
+import Test.E2E.Support.Env (E2ESpec)
+import Test.Spec as Spec
+
+spec :: E2ESpec
+spec = do
+  Spec.describe "check if there's a new compiler" do
+    Spec.it "enqueues matrix jobs for packages with no dependencies when new compiler detected" do
+      -- The test env has compilers 0.15.10 and 0.15.11 available.
+      -- [email protected] fixture only has compiler 0.15.10 in metadata.
+      -- So 0.15.11 should be detected as "new" at startup, triggering
+      -- matrix jobs for packages with no dependencies.
+      jobs <- Client.getJobs
+      let
+        isNewCompilerMatrixJob :: Job -> Boolean
+        isNewCompilerMatrixJob = case _ of
+          MatrixJob { compilerVersion } ->
+            compilerVersion == Utils.unsafeVersion "0.15.11"
+          _ -> false
+
+        matrixJobs = Array.filter isNewCompilerMatrixJob jobs
+
+        -- Get package names from matrix jobs
+        matrixPackages = matrixJobs # Array.mapMaybe case _ of
+          MatrixJob { packageName } -> Just packageName
+          _ -> Nothing
+
+      -- Should have matrix jobs for packages with no dependencies
+      -- prelude has no dependencies, so it should get a matrix job
+      let preludeName = Utils.unsafePackageName "prelude"
+      unless (Array.elem preludeName matrixPackages) do
+        Assert.fail $ "Expected matrix job for prelude with compiler 0.15.11, found: "
+          <> show (Array.length matrixJobs)
+          <> " matrix jobs for packages: "
+          <> String.joinWith ", " (map PackageName.print matrixPackages)

app-e2e/src/Test/E2E/Scripts.purs

@@ -0,0 +1,257 @@
+-- | End-to-end tests for the cronjob scripts:
+-- | - DailyImporter: Detects new package versions via GitHub tags
+-- | - PackageSetUpdater: Detects recent uploads for package set inclusion
+-- | - PackageTransferrer: Detects packages that moved to new GitHub locations
+-- |
+-- | These tests verify that the scripts properly enqueue jobs via the API.
+module Test.E2E.Scripts (spec) where
+
+import Registry.App.Prelude
+
+import Control.Monad.Reader (ask)
+import Data.Array as Array
+import Data.Map as Map
+import Effect.Aff as Aff
+import Node.Path as Path
+import Node.Process as Process
+import Registry.API.V1 (Job(..))
+import Registry.App.CLI.Git as Git
+import Registry.App.Effect.Cache as Cache
+import Registry.App.Effect.Env as Env
+import Registry.App.Effect.GitHub as GitHub
+import Registry.App.Effect.Log as Log
+import Registry.App.Effect.Registry as Registry
+import Registry.Foreign.Octokit as Octokit
+import Registry.Location (Location(..))
+import Registry.Operation (AuthenticatedPackageOperation(..))
+import Registry.Operation as Operation
+import Registry.PackageName as PackageName
+import Registry.Scripts.DailyImporter as DailyImporter
+import Registry.Scripts.PackageSetUpdater as PackageSetUpdater
+import Registry.Scripts.PackageTransferrer as PackageTransferrer
+import Registry.Test.Assert as Assert
+import Registry.Test.Utils as Utils
+import Registry.Version as Version
+import Run as Run
+import Run.Except as Except
+import Test.E2E.Support.Client as Client
+import Test.E2E.Support.Env (E2E, E2ESpec)
+import Test.Spec as Spec
+
+-- | Constants for repeated package names and versions in tests
+typeEqualityName :: PackageName.PackageName
+typeEqualityName = Utils.unsafePackageName "type-equality"
+
+typeEqualityV401 :: Version.Version
+typeEqualityV401 = Utils.unsafeVersion "4.0.1"
+
+typeEqualityV402 :: Version.Version
+typeEqualityV402 = Utils.unsafeVersion "4.0.2"
+
+compiler01510 :: Version.Version
+compiler01510 = Utils.unsafeVersion "0.15.10"
+
+spec :: E2ESpec
+spec = do
+  Spec.describe "DailyImporter" do
+    Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do
+      runDailyImporterScript
+      jobs <- Client.getJobs
+
+      -- type-equality has v4.0.1 published but v4.0.2 in tags (per wiremock config)
+      let
+        isTypeEqualityPublishJob :: Job -> Boolean
+        isTypeEqualityPublishJob = case _ of
+          PublishJob { packageName, packageVersion } ->
+            packageName == typeEqualityName && packageVersion == typeEqualityV402
+          _ -> false
+
+        typeEqualityJob = Array.find isTypeEqualityPublishJob jobs
+
+      case typeEqualityJob of
+        Just (PublishJob { payload }) -> do
+          -- Verify compiler is either Nothing (API will discover) or Just 0.15.10
+          case payload.compiler of
+            Nothing -> pure unit
+            Just c | c /= compiler01510 ->
+              Assert.fail $ "Expected compiler 0.15.10 or Nothing but got " <> Version.print c
+            _ -> pure unit
+        Just _ -> Assert.fail "Expected PublishJob but got different job type"
+        Nothing -> do
+          let publishJobs = Array.filter isPublishJob jobs
+          Assert.fail $ "Expected to find a publish job for [email protected] but found "
+            <> show (Array.length publishJobs)
+            <> " publish jobs: "
+            <> show (map formatPublishJob publishJobs)
+
+    Spec.it "does not enqueue jobs for already-published versions" do
+      runDailyImporterScript
+      jobs <- Client.getJobs
+
+      -- type-equality v4.0.1 is already published, should NOT have a new job
+      let
+        isDuplicateJob :: Job -> Boolean
+        isDuplicateJob = case _ of
+          PublishJob { packageName, packageVersion } ->
+            packageName == typeEqualityName && packageVersion == typeEqualityV401
+          _ -> false
+
+        duplicateJob = Array.find isDuplicateJob jobs
+
+      case duplicateJob of
+        Nothing -> pure unit -- Good, no duplicate job
+        Just _ -> Assert.fail "Found unexpected publish job for already-published [email protected]"
+
+  Spec.describe "PackageTransferrer" do
+    Spec.it "enqueues transfer jobs when package location changes" do
+      runPackageTransferrerScript
+      -- type-equality metadata says purescript, but tags point to new-owner
+      jobs <- Client.getJobs
+      let
+        isTypeEqualityTransferJob :: Job -> Boolean
+        isTypeEqualityTransferJob = case _ of
+          TransferJob { packageName } ->
+            packageName == typeEqualityName
+          _ -> false
+      case Array.find isTypeEqualityTransferJob jobs of
+        Just (TransferJob { packageName, payload }) -> do
+          -- Verify packageName
+          when (packageName /= typeEqualityName) do
+            Assert.fail $ "Wrong package name: " <> PackageName.print packageName
+          -- Verify newLocation in payload
+          case payload.payload of
+            Transfer { newLocation } ->
+              case newLocation of
+                GitHub { owner } ->
+                  when (owner /= "new-owner") do
+                    Assert.fail $ "Expected owner 'new-owner' but got '" <> owner <> "'"
+                _ -> Assert.fail "Expected GitHub location"
+            _ -> Assert.fail "Expected Transfer payload"
+        Just _ -> Assert.fail "Expected TransferJob but got different job type"
+        Nothing -> do
+          let transferJobs = Array.filter isTransferJob jobs
+          Assert.fail $ "Expected to find a transfer job for 'type-equality' but found "
+            <> show (Array.length transferJobs)
+            <> " transfer jobs"
+
+  Spec.describe "PackageSetUpdater" do
+    Spec.it "enqueues package set update for recent uploads not in set" do
+      runPackageSetUpdaterScript
+      jobs <- Client.getJobs
+      let packageSetJobs = Array.filter isPackageSetJob jobs
+      case Array.head packageSetJobs of
+        Just (PackageSetJob { payload }) ->
+          case payload of
+            Operation.PackageSetUpdate { packages } ->
+              case Map.lookup typeEqualityName packages of
+                Just (Just _) -> pure unit
+                _ -> Assert.fail "Expected type-equality in package set update"
+        Just _ -> Assert.fail "Expected PackageSetJob but got different job type"
+        Nothing -> Assert.fail "Expected package set job to be enqueued"
+
+-- | Common environment for running registry scripts in E2E tests
+type ScriptSetup =
+  { privateKey :: String
+  , resourceEnv :: Env.ResourceEnv
+  , registryEnv :: Registry.RegistryEnv
+  , octokit :: Octokit.Octokit
+  , cache :: FilePath
+  , githubCacheRef :: Cache.CacheRef
+  }
+
+-- | Set up common environment for running registry scripts
+setupScript :: E2E ScriptSetup
+setupScript = do
+  { stateDir, privateKey } <- ask
+  liftEffect $ Process.chdir stateDir
+  resourceEnv <- liftEffect Env.lookupResourceEnv
+  token <- liftEffect $ Env.lookupRequired Env.githubToken
+  githubCacheRef <- liftAff Cache.newCacheRef
+  registryCacheRef <- liftAff Cache.newCacheRef
+  let cache = Path.concat [ stateDir, "scratch", ".cache" ]
+  octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl
+  debouncer <- liftAff Registry.newDebouncer
+  let
+    registryEnv :: Registry.RegistryEnv
+    registryEnv =
+      { pull: Git.Autostash
+      , write: Registry.ReadOnly
+      , repos: Registry.defaultRepos
+      , workdir: Path.concat [ stateDir, "scratch" ]
+      , debouncer
+      , cacheRef: registryCacheRef
+      }
+  pure { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef }
+
+-- | Run the DailyImporter script in Submit mode
+runDailyImporterScript :: E2E Unit
+runDailyImporterScript = do
+  { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript
+  result <- liftAff
+    $ DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl
+    # Except.runExcept
+    # Registry.interpret (Registry.handle registryEnv)
+    # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef })
+    # Log.interpret (Log.handleTerminal Quiet)
+    # Env.runResourceEnv resourceEnv
+    # Run.runBaseAff'
+  case result of
+    Left err -> liftAff $ Aff.throwError $ Aff.error $ "DailyImporter failed: " <> err
+    Right _ -> pure unit
+
+-- | Run the PackageTransferrer script in Submit mode
+runPackageTransferrerScript :: E2E Unit
+runPackageTransferrerScript = do
+  { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript
+  result <- liftAff
+    $ PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl
+    # Except.runExcept
+    # Registry.interpret (Registry.handle registryEnv)
+    # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef })
+    # Log.interpret (Log.handleTerminal Quiet)
+    # Env.runResourceEnv resourceEnv
+    # Run.runBaseAff'
+  case result of
+    Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageTransferrer failed: " <> err
+    Right _ -> pure unit
+
+-- | Run the PackageSetUpdater script in Submit mode
+runPackageSetUpdaterScript :: E2E Unit
+runPackageSetUpdaterScript = do
+  { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript
+  result <- liftAff
+    $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit resourceEnv.registryApiUrl
+    # Except.runExcept
+    # Registry.interpret (Registry.handle registryEnv)
+    # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef })
+    # Log.interpret (Log.handleTerminal Quiet)
+    # Env.runResourceEnv resourceEnv
+    # Run.runBaseAff'
+  case result of
+    Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageSetUpdater failed: " <> err
+    Right _ -> pure unit
+
+-- | Check if a job is a PublishJob
+isPublishJob :: Job -> Boolean
+isPublishJob = case _ of
+  PublishJob _ -> true
+  _ -> false
+
+-- | Format a PublishJob for debugging output
+formatPublishJob :: Job -> String
+formatPublishJob = case _ of
+  PublishJob { packageName, packageVersion } ->
+    PackageName.print packageName <> "@" <> Version.print packageVersion
+  _ -> "<not a publish job>"
+
+-- | Check if a job is a TransferJob
+isTransferJob :: Job -> Boolean
+isTransferJob = case _ of
+  TransferJob _ -> true
+  _ -> false
+
+-- | Check if a job is a PackageSetJob
+isPackageSetJob :: Job -> Boolean
+isPackageSetJob = case _ of
+  PackageSetJob _ -> true
+  _ -> false