Integrate OpenSearch

.vscode/settings.json

@@ -30,7 +30,7 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[dockercompose]": {
-    "editor.defaultFormatter": "ms-azuretools.vscode-docker"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[dockerfile]": {
     "editor.defaultFormatter": "ms-azuretools.vscode-docker"

Dockerfile

@@ -44,7 +44,7 @@ RUN  mkdir /tmp/vespa-cli \
     && cp -r /tmp/vespa-cli/share/* /usr/local/share/ \
     && rm -rf /tmp/vespa-cli.tar.gz /tmp/vespa-cli
 
-# Download embedding model, which gets deployed by setup_vespa.py
+# Download embedding model, which gets deployed by Django vespa management command (see vespa.py)
 ADD "https://github.com/vespa-engine/sample-apps/raw/master/simple-semantic-search/model/e5-small-v2-int8.onnx" /opt/models/model.onnx
 ADD "https://raw.githubusercontent.com/vespa-engine/sample-apps/master/simple-semantic-search/model/tokenizer.json" /opt/models/tokenizer.json
 

TODO.md

@@ -2,6 +2,7 @@
 
 ## High Priority
 
+- Rename patient_birth_date to patient_birthdate (sounds better, but DICOM uses Patient Birth Date)
 - Check if for RAG ranking should be turned off for performance improvements (and using some fixed sort order)
 - Some present provider.max_results to the user somehow, especially important if the query results (step 1) is larger
 - task control panel

compose/docker-compose.base.yml

@@ -1,15 +1,16 @@
-version: "3.8"
-
 x-app: &default-app
   volumes:
     - /mnt:/mnt
   depends_on:
     - postgres
-    - vespa
   environment:
     USE_DOCKER: 1
     DJANGO_STATIC_ROOT: "/var/www/radis/static/"
     DATABASE_URL: "psql://postgres:[email protected]:5432/postgres"
+    OPENSEARCH_ENABLED: "${OPENSEARCH_ENABLED-true}"
+    OPENSEARCH_HOST: "opensearch-node1.local"
+    OPENSEARCH_PORT: "9200"
+    VESPA_ENABLED: "${VESPA_ENABLED-false}"
     VESPA_HOST: "vespa.local"
     VESPA_CONFIG_PORT: "19071"
     VESPA_DATA_PORT: "8080"
@@ -57,28 +58,12 @@ services:
         celery --broker=amqp://rabbit/ flower --url_prefix=flower
       "
 
-  llamacpp:
-    image: ghcr.io/ggerganov/llama.cpp:server
-    hostname: llamacpp.local
-
   postgres:
     image: postgres:16.2
     hostname: postgres.local
     volumes:
       - postgres_data:/var/lib/postgresql/data
 
-  vespa:
-    image: vespaengine/vespa:8
-    hostname: vespa.local
-    healthcheck:
-      test: curl http://localhost:19071/state/v1/health
-      timeout: 10s
-      retries: 3
-      start_period: 40s
-    volumes:
-      - vespa_data:/opt/vespa/var
-      - vespa_logs:/opt/vespa/logs
-
   # RabbitMQ authentication can't be disabled. So when we try to log into
   # the management console we have to use "guest" as username and password.
   # The real authentication happens by ADIT itself, because the management
@@ -105,7 +90,5 @@ volumes:
   radis_data:
   flower_data:
   postgres_data:
-  vespa_data:
-  vespa_logs:
   rabbit_data:
   redis_data:

compose/docker-compose.dev.yml

@@ -1,5 +1,3 @@
-version: "3.8"
-
 x-app: &default-app
   image: radis_dev:latest
   volumes:
@@ -29,85 +27,121 @@ services:
         wait-for-it -s postgres.local:5432 -t 60 && 
         ./manage.py migrate &&
         ./manage.py collectstatic --no-input &&
-        wait-for-it -s vespa.local:19071 -t 60 &&
-        ./manage.py setup_vespa --generate --deploy &&
+        if [[ $$OPENSEARCH_ENABLED == true ]]; then
+          wait-for-it -s opensearch-node1.local:9200 -t 60 &&
+          ./manage.py opensearch --mappings dev
+        fi
+        if [[ $$VESPA_ENABLED == true ]]; then
+          wait-for-it -s vespa.local:19071 -t 60 &&
+          ./manage.py vespa --generate --deploy
+        fi
         ./manage.py populate_users_and_groups --users 20 --groups 3 &&
         ./manage.py populate_reports --report-language de &&
         ./manage.py runserver 0.0.0.0:8000
       "
-    profiles:
-      - full
-      - web
 
   worker_default:
     <<: *default-app
     command: |
       ./manage.py celery_worker -c 1 -Q default_queue --autoreload
-    profiles:
-      - full
 
   worker_vespa:
     <<: *default-app
     command: |
       ./manage.py celery_worker -c 1 -Q vespa_queue --autoreload
-    profiles:
-      - full
+    profiles: ["vespa"]
 
   worker_llm:
     <<: *default-app
     command: |
       ./manage.py celery_worker -c 1 -Q llm_queue --autoreload
-    profiles:
-      - full
 
   celery_beat:
     <<: *default-app
     command: |
       ./manage.py celery_beat --autoreload
-    profiles:
-      - full
 
   flower:
     <<: *default-app
-    profiles:
-      - full
 
-  llamacpp:
+  llamacpp-cpu:
+    image: ghcr.io/ggerganov/llama.cpp:server
+    hostname: llamacpp.local
     ports:
       - 9610:8080
     volumes:
       - ../models:/models
     command: "-m /models/model.gguf -c 512 --host 0.0.0.0 --port 8080"
-    profiles:
-      - full
+    profiles: ["cpu"]
+
+  llamacpp-gpu:
+    image: ghcr.io/ggerganov/llama.cpp:server-cuda
+    hostname: llamacpp.local
+    ports:
+      - 9610:8080
+    volumes:
+      - ../models:/models
+    command: "-m /models/model.gguf -c 512 --host 0.0.0.0 --port 8080"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+    profiles: ["gpu"]
 
   postgres:
     environment:
       POSTGRES_PASSWORD: "postgres"
-    profiles:
-      - full
-      - web
+
+  # https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/#sample-docker-composeyml
+  opensearch-node1:
+    image: opensearchproject/opensearch:latest
+    hostname: opensearch-node1.local
+    volumes:
+      - opensearch_data1:/usr/share/opensearch/data
+    environment:
+      OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
+      DISABLE_SECURITY_PLUGIN: "true"
+      discovery.type: single-node # https://github.com/gitpod-io/gitpod/issues/8399
+    ports:
+      - 9200:9200
+    profiles: ["opensearch"]
+
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:latest
+    hostname: opensearch-dashboards.local
+    ports:
+      - 5601:5601
+    environment:
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
+      OPENSEARCH_HOSTS: '["http://opensearch-node1.local:9200"]'
+    profiles: ["opensearch"]
 
   vespa:
+    image: vespaengine/vespa:8
+    hostname: vespa.local
+    healthcheck:
+      test: curl http://localhost:19071/state/v1/health
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    volumes:
+      - vespa_data:/opt/vespa/var
+      - vespa_logs:/opt/vespa/logs
     environment:
       # Reduce memory usage of Vespa during development (espcially on Cloud IDEs), see
       # https://docs.vespa.ai/en/operations/node-setup.html#memory-settings
       VESPA_CONFIGSERVER_JVMARGS: "-Xms32M -Xmx128M"
       VESPA_CONFIGPROXY_JVMARGS: "-Xms32M -Xmx32M"
     ports:
       - 9620:8080
-    profiles:
-      - full
-      - web
-
-  rabbit:
-    profiles:
-      - full
-
-  redis:
-    profiles:
-      - full
+    profiles: ["vespa"]
 
 volumes:
   vscode-server:
   vscode-server-insiders:
+  opensearch_data1:
+  vespa_data:
+  vespa_logs: