Merge pull request #11 from openradx/switch-to-procrastinate

Switch to procrastinate

Dockerfile

@@ -14,7 +14,7 @@ ENV PYTHONUNBUFFERED=1 \
     # poetry
     # https://python-poetry.org/docs/#installing-with-the-official-installer
     # https://python-poetry.org/docs/configuration/#using-environment-variables
-    POETRY_VERSION=1.8.2 \
+    POETRY_VERSION=1.8.3 \
     # make poetry install to this location
     POETRY_HOME="/opt/poetry" \
     # make poetry create the virtual environment in the project's root
@@ -72,10 +72,9 @@ RUN poetry install
 RUN playwright install --with-deps chromium
 
 # Required folders for RADIS
-RUN mkdir -p /var/www/radis/logs \
-    /var/www/radis/static \
-    /var/www/radis/ssl \
-    /var/www/radis/celery
+RUN mkdir -p /var/www/web/logs \
+    /var/www/web/static \
+    /var/www/web/ssl
 
 # will become mountpoint of our code
 WORKDIR /app
@@ -87,9 +86,8 @@ COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
 COPY . /app/
 
 # Required folders for RADIS
-RUN mkdir -p /var/www/radis/logs \
-    /var/www/radis/static \
-    /var/www/radis/ssl \
-    /var/www/radis/celery
+RUN mkdir -p /var/www/web/logs \
+    /var/www/web/static \
+    /var/www/web/ssl
 
 WORKDIR /app

TODO.md

@@ -14,7 +14,6 @@
   - Reference: name (unique), match (unique)
 - Remove unneeded templatetags
 - Are pandas and openpyxl needed as deps?!
-- Remove Redis if not needed anymore
 
 ## Fix
 
@@ -91,5 +90,4 @@
 - Delete reset_dev_db and add reset option to populate_dev_db
 - globals.d.ts
 - rename all Alpine components to Uppercase
-- Turn off debug logging in Celery
 - Add metaclass=ABCMeta to abstract core/models and core/views (also core/tables and core/filters even in RADIS)

compose/docker-compose.base.yml

@@ -5,22 +5,16 @@ x-app: &default-app
     - postgres
   environment:
     USE_DOCKER: 1
-    DJANGO_STATIC_ROOT: "/var/www/radis/static/"
+    DJANGO_STATIC_ROOT: "/var/www/web/static/"
     DATABASE_URL: "psql://postgres:[email protected]:5432/postgres"
-    RABBITMQ_URL: "amqp://rabbit"
-    RABBIT_MANAGEMENT_HOST: "rabbit"
-    RABBIT_MANAGEMENT_PORT: "15672"
-    REDIS_URL: "redis://redis.local:6379/0"
     LLAMACPP_URL: "http://llamacpp.local:8080"
-    FLOWER_HOST: "flower.local"
-    FLOWER_PORT: "5555"
 
 services:
   init:
     <<: *default-app
     hostname: init.local
     volumes:
-      - radis_data:/var/www/radis
+      - web_data:/var/www/web
       - /mnt:/mnt
 
   web:
@@ -29,7 +23,7 @@ services:
     build:
       context: ..
     volumes:
-      - radis_data:/var/www/radis
+      - web_data:/var/www/web
       - /mnt:/mnt
 
   worker_default:
@@ -40,50 +34,12 @@ services:
     <<: *default-app
     hostname: worker_llm.local
 
-  celery_beat:
-    <<: *default-app
-    hostname: celery_beat.local
-
-  flower:
-    <<: *default-app
-    hostname: flower.local
-    command: >
-      bash -c "
-        wait-for-it -s rabbit:5672 -t 100 &&
-        celery --broker=amqp://rabbit/ flower --url_prefix=flower
-      "
-
   postgres:
     image: postgres:16
     hostname: postgres.local
     volumes:
       - postgres_data:/var/lib/postgresql/data
 
-  # RabbitMQ authentication can't be disabled. So when we try to log into
-  # the management console we have to use "guest" as username and password.
-  # The real authentication happens by ADIT itself, because the management
-  # console is behind a ProxyView.
-  rabbit:
-    image: rabbitmq:3.12.2-management
-    configs:
-      - source: rabbit_config
-        target: /etc/rabbitmq/rabbitmq.conf
-    volumes:
-      - rabbit_data:/var/lib/rabbitmq
-
-  redis:
-    image: redis:7.2
-    hostname: redis.local
-    volumes:
-      - redis_data:/data
-
-configs:
-  rabbit_config:
-    file: ./rabbitmq/rabbitmq.conf
-
 volumes:
-  radis_data:
-  flower_data:
+  web_data:
   postgres_data:
-  rabbit_data:
-  redis_data:

compose/docker-compose.dev.yml

@@ -50,21 +50,19 @@ services:
 
   worker_default:
     <<: *default-app
-    command: |
-      ./manage.py celery_worker -c 1 -Q default_queue --autoreload
+    command: >
+      bash -c "
+        wait-for-it -s postgres.local:5432 -t 60 &&
+        ./manage.py worker -l debug -q default --autoreload
+      "
 
   worker_llm:
     <<: *default-app
-    command: |
-      ./manage.py celery_worker -c 1 -Q llm_queue --autoreload
-
-  celery_beat:
-    <<: *default-app
-    command: |
-      ./manage.py celery_beat --autoreload
-
-  flower:
-    <<: *default-app
+    command: >
+      bash -c "
+        wait-for-it -s postgres.local:5432 -t 60 &&
+        ./manage.py worker -l debug -q llm --autoreload
+      "
 
   llamacpp_cpu:
     <<: *llamacpp

compose/docker-compose.prod.yml

@@ -5,8 +5,8 @@ x-app: &default-app
   environment:
     ENABLE_REMOTE_DEBUGGING: 0
     DJANGO_SETTINGS_MODULE: "radis.settings.production"
-    SSL_CERT_FILE: "/var/www/radis/ssl/cert.pem"
-    SSL_KEY_FILE: "/var/www/radis/ssl/key.pem"
+    SSL_CERT_FILE: "/var/www/web/ssl/cert.pem"
+    SSL_KEY_FILE: "/var/www/web/ssl/key.pem"
 
 x-deploy: &deploy
   replicas: 1
@@ -44,32 +44,29 @@ services:
       bash -c "
         wait-for-it -s init.local:8000 -t 300 &&
         echo 'Starting web server ...'
-        daphne -b 0.0.0.0 -p 80 -e ssl:443:privateKey=/var/www/radis/ssl/key.pem:certKey=/var/www/radis/ssl/cert.pem radis.asgi:application
+        daphne -b 0.0.0.0 -p 80 -e ssl:443:privateKey=/var/www/web/ssl/key.pem:certKey=/var/www/web/ssl/cert.pem radis.asgi:application
       "
     deploy:
       <<: *deploy
       replicas: 3
 
   worker_default:
     <<: *default-app
-    command: ./manage.py celery_worker -Q default_queue
+    command: >
+      bash -c "
+        wait-for-it -s postgres.local:5432 -t 60 &&
+        ./manage.py worker -q default
+      "
     deploy:
       <<: *deploy
 
   worker_llm:
     <<: *default-app
-    command: ./manage.py celery_worker -c 1 -Q llm_queue
-    deploy:
-      <<: *deploy
-
-  celery_beat:
-    <<: *default-app
-    command: ./manage.py celery_beat
-    deploy:
-      <<: *deploy
-
-  flower:
-    <<: *default-app
+    command: >
+      bash -c "
+        wait-for-it -s postgres.local:5432 -t 60 &&
+        ./manage.py worker -q llm
+      "
     deploy:
       <<: *deploy
 
@@ -101,13 +98,5 @@ services:
     deploy:
       <<: *deploy
 
-  rabbit:
-    deploy:
-      <<: *deploy
-
-  redis:
-    deploy:
-      <<: *deploy
-
 volumes:
   models_data: