Upgrade to Celery 5.0

docker-compose.yml

@@ -2,90 +2,89 @@
 version: "3.7"
 
 secrets:
-  redis_conf:
-    file: ./secrets/redis.conf
   admiral_yml:
     file: ./secrets/admiral.yml
   mongo_root_passwd_txt:
     file: ./secrets/mongo-root-passwd.txt
+  redis_conf:
+    file: ./secrets/redis.conf
 
 x-admiral-template: &admiral-template
   build:
     context: .
     dockerfile: Dockerfile-admiral
-  image: admiral
-  init: true
+  deploy:
+    mode: replicated
+    replicas: 6
   environment:
     ADMIRAL_CONFIG_FILE: "/run/secrets/admiral.yml"
     ADMIRAL_CONFIG_SECTION: dev-mode
     ADMIRAL_WORKER_NAME: dev
+  image: admiral
+  init: true
   secrets:
     - source: admiral_yml
       target: admiral.yml
   volumes:  # map for development only, comment out otherwise
     - ./src/admiral:/usr/src/admiral/admiral
-  deploy:
-    mode: replicated
-    replicas: 6
 
 services:
-  redis:
-    image: "redis:alpine"
-    command:
-      - "redis-server"
-      - "/run/secrets/redis_conf"
+  celery-flower:
+    environment:
+      # TODO: https://github.com/cisagov/admiral/issues/19
+      CELERY_BROKER_URL: "redis://:fruitcake@redis:6379/0"
+    image: crgwbr/docker-celery-flower
     ports:
-      - "6379:6379"
-    secrets:
-      - source: redis_conf
+      - "5555:5555"
 
-  redis-commander:
-    hostname: redis-commander
-    image: rediscommander/redis-commander:latest
-    # build: .
-    restart: always
-    depends_on:
-      - redis
+  cert-worker:
+    <<: *admiral-template
     environment:
-      - REDIS_HOSTS=default:redis:6379:0:fruitcake
-    ports:
-      - "8082:8081"
+      ADMIRAL_CONFIG_SECTION: cert-worker
+      ADMIRAL_WORKER_NAME: cert
 
   mongo:
-    image: mongo
-    restart: always
     environment:
       MONGO_INITDB_DATABASE: certs
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/mongo_root_passwd_txt
+    image: mongo:4.2
+    restart: always
     secrets:
       - source: mongo_root_passwd_txt
     volumes:
       - ./init/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro
 
   mongo-express:
-    image: mongo-express
-    restart: always
-    ports:
-      - 8083:8081
     environment:
       ME_CONFIG_MONGODB_SERVER: mongo
       ME_CONFIG_MONGODB_ADMINUSERNAME: root
       ME_CONFIG_MONGODB_ADMINPASSWORD: example
+    image: mongo-express:0.54
+    ports:
+      - "8083:8081"
+    restart: always
 
-  celery-flower:
-    image: crgwbr/docker-celery-flower
-    environment:
-      # TODO: get this into the secrets raft
-      CELERY_BROKER_URL: "redis://:fruitcake@redis:6379/0"
+  redis:
+    command:
+      - "redis-server"
+      - "/run/secrets/redis_conf"
+    image: redis:7.0-alpine
     ports:
-      - "5555:5555"
+      - "6379:6379"
+    secrets:
+      - source: redis_conf
 
-  cert-worker:
-    <<: *admiral-template
+  redis-commander:
+    depends_on:
+      - redis
     environment:
-      ADMIRAL_CONFIG_SECTION: cert-worker
-      ADMIRAL_WORKER_NAME: cert
+      REDIS_HOSTS: "default:redis:6379:0:fruitcake"
+    hostname: redis-commander
+    image: ghcr.io/joeferner/redis-commander:0.8
+    ports:
+      - "8082:8081"
+    restart: always
 
   scanner-worker:
     <<: *admiral-template

setup.py

@@ -88,10 +88,7 @@ def get_version(version_file):
     py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
     include_package_data=True,
     install_requires=[
-        # Celery 5.0 introduces breaking changes to the CLI. See
-        # https://docs.celeryq.dev/en/stable/history/whatsnew-5.0.html?highlight=command%20line%20options#new-command-line-interface
-        # for more information.
-        "celery >= 4.3.0,<5.0.0",
+        "celery >= 5.0.0,<6",
         "cryptography >= 2.4.2",
         "defusedxml",
         "dnspython",

src/admiral/celery.py

@@ -128,12 +128,11 @@ def main():
     # https://celery.readthedocs.io/en/latest/userguide/extending.html#adding-new-command-line-options
 
     if args["--interactive"]:
-        celery.start(argv=["celery", "-A", "admiral", "shell"])
+        celery.start(argv=["-A", "admiral", "shell"])
     else:
         worker_name = os.environ.get(WORKER_NAME_ENV_KEY, "unnamed")
         celery.start(
             argv=[
-                "celery",
                 "-A",
                 "admiral",
                 "worker",