[WIP] Add support for OpenSearch as a vector database

.github/workflows/pr-chart-validate.yaml

@@ -60,4 +60,5 @@ jobs:
           done
           ct lint ${exclude_param} --check-version-increment=false --validate-maintainers=false --chart-dirs ${{ env.CHARTS_DIRS }} --target-branch ${{ github.event.repository.default_branch }} \
               --chart-repos milvus=https://zilliztech.github.io/milvus-helm \
-              --chart-repos qdrant=https://qdrant.github.io/qdrant-helm
+              --chart-repos qdrant=https://qdrant.github.io/qdrant-helm \
+              --chart-repos opensearch=https://opensearch-project.github.io/helm-charts

helm-charts/chatqna/Chart.yaml

@@ -63,5 +63,10 @@ dependencies:
   - name: nginx
     version: 0-latest
     repository: "file://../common/nginx"
+  - name: opensearch
+    version: 2.31.0
+    repository: https://opensearch-project.github.io/helm-charts
+    condition: opensearch.enabled
+
 version: 0-latest
 appVersion: "v1.0"

helm-charts/chatqna/README.md

@@ -26,6 +26,11 @@ export MODELDIR="/mnt/opea-models"
 export MODELNAME="meta-llama/Meta-Llama-3-8B-Instruct"
 # To use CPU with vLLM
 helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME}
+# To use CPU with vLLM and OpenSearch vector DB
+# Admin password must be at least 8 characters, and contain at least one uppercase letter, lowercase letter, digit, and special character
+export OPENSEARCH_INITIAL_ADMIN_PASSWORD="insert-initial-opensearch-admin-password"
+helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} --set data-prep.OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} --set retriever-usvc.OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} --set "opensearch.extraEnvs[0].name=OPENSEARCH_INITIAL_ADMIN_PASSWORD,opensearch.extraEnvs[0].value=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}" -f chatqna/cpu-opensearch-values.yaml
+
 # To use Gaudi device with vLLM
 #helm install chatqna chatqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set vllm.LLM_MODEL_ID=${MODELNAME} -f chatqna/gaudi-vllm-values.yaml
 # To use CPU with TGI

helm-charts/chatqna/cpu-opensearch-values.yaml

@@ -0,0 +1,36 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+image:
+  repository: opea/chatqna
+redis-vector-db:
+  enabled: false
+
+# Common fields between data-prep and retriever-usvc must match
+# to ensure that RAG works properly
+data-prep:
+  DATAPREP_BACKEND: "OPENSEARCH"
+  OPENSEARCH_HOST: "opensearch-cluster-master"
+  OPENSEARCH_PORT: 9200
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: "longPassWithNumCharCapLower1#"
+  OPENSEARCH_SSL: true
+  INDEX_NAME: "rag_opensearch"
+  KEY_INDEX_NAME: "file-keys"
+  TIMEOUT_SECONDS: 600
+  SEARCH_BATCH_SIZE: 10
+retriever-usvc:
+  RETRIEVER_BACKEND: "OPENSEARCH"
+  OPENSEARCH_HOST: "opensearch-cluster-master"
+  OPENSEARCH_PORT: 9200
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: "longPassWithNumCharCapLower1#"
+  OPENSEARCH_SSL: true
+  OPENSEARCH_INDEX_NAME: "rag_opensearch"
+
+opensearch:
+  enabled: true
+  persistence:
+    enabled: false
+  singleNode: true
+  extraEnvs:
+    - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+      value: "longPassWithNumCharCapLower1#" # must match value above or data-prep cannot access OpenSearch

helm-charts/chatqna/values.yaml

@@ -92,6 +92,8 @@ teirerank:
 # vector db choice, redis by default.
 redis-vector-db:
   enabled: true
+opensearch:
+  enabled: false
 # Microservice layer, disabled by default
 llm-uservice:
   enabled: false

helm-charts/common/data-prep/Chart.yaml

@@ -28,3 +28,7 @@ dependencies:
   - name: commonlib
     version: 0-latest
     repository: "file://../commonlib"
+  - name: opensearch
+    version: 2.31.0
+    repository: https://opensearch-project.github.io/helm-charts
+    condition: opensearch.enabled

helm-charts/common/data-prep/README.md

@@ -14,6 +14,8 @@ Helm chart for deploying OPEA data-prep microservice.
 
 - Qdrant DB: please refer to [qdrant-helm](https://github.com/qdrant/qdrant-helm/tree/qdrant-1.13.1/charts/qdrant) for more information.
 
+- OpenSearch DB: please refer to [opensearch-helm](https://artifacthub.io/packages/helm/opensearch-project-helm-charts/opensearch) for more information.
+
 First, you need to install the `tei` helm chart and one of the vector DB service, i.e. `redis-vector-db` chart.
 
 After you've deployed dependency charts successfully, please run `kubectl get svc` to get the service endpoint URL respectively, i.e. `http://tei:80`, `redis://redis-vector-db:6379`.
@@ -23,7 +25,7 @@ To install `data-prep` chart, run the following:
 ```console
 cd GenAIInfra/helm-charts/common/data-prep
 helm dependency update
-export HFTOKEN="insert-your-huggingface-token-here"
+export HF_TOKEN="insert-your-huggingface-token-here"
 export TEI_EMBEDDING_ENDPOINT="http://tei"
 
 # Install data-prep with Redis DB backend
@@ -40,6 +42,12 @@ helm install data-prep . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT}
 # export DATAPREP_BACKEND="QDRANT"
 # export DB_HOST="qdrant"
 # helm install data-prep . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} --set global.HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} --set DATAPREP_BACKEND=${DATAPREP_BACKEND} --set QDRANT_HOST=${DB_HOST},QDRANT_PORT=6333,COLLECTION_NAME=rag_qdrant
+
+# Install data-prep with OpenSearch DB backend
+# export DATAPREP_BACKEND="OPENSEARCH"
+# export DB_HOST="opensearch-cluster-master"
+# export OPENSEARCH_INITIAL_ADMIN_PASSWORD="insert-your-initial-admin-password"
+# helm install data-prep . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} --set global.HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} --set DATAPREP_BACKEND=${DATAPREP_BACKEND} --set OPENSEARCH_HOST=${DB_HOST} --set OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} --set "opensearch.extraEnvs[0].name=OPENSEARCH_INITIAL_ADMIN_PASSWORD,opensearch.extraEnvs[0].value=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}" -f opensearch-values.yaml
 ```
 
 ## Verify
@@ -59,16 +67,17 @@ curl http://localhost:6007/v1/dataprep/ingest  \
 
 ## Values
 
-| Key                             | Type   | Default   | Description                                                                                             |
-| ------------------------------- | ------ | --------- | ------------------------------------------------------------------------------------------------------- |
-| service.port                    | string | `"6007"`  |                                                                                                         |
-| global.HUGGINGFACEHUB_API_TOKEN | string | `""`      | Your own Hugging Face API token                                                                         |
-| DATAPREP_BACKEND                | string | `"REDIS"` | vector DB backend to use, one of "REDIS", "MILVUS", "QDRANT"                                            |
-| REDIS_HOST                      | string | `""`      | Redis service URL host, only valid for Redis, please see `values.yaml` for other Redis configuration    |
-| MILVUS_HOST                     | string | `""`      | Milvus service URL host, only valid for Milvus, please see `values.yaml` for other Milvus configuration |
-| QDRANT_HOST                     | string | `""`      | Qdrant service URL host, only valid for Qdrant, please see `values.yaml` for other Qdrant configuration |
-| TEI_EMBEDDING_ENDPOINT          | string | `""`      |                                                                                                         |
-| global.monitoring               | bool   | `false`   | See ../../monitoring.md before enabling!                                                                |
+| Key                             | Type   | Default   | Description                                                                                                 |
+| ------------------------------- | ------ | --------- | ----------------------------------------------------------------------------------------------------------- |
+| service.port                    | string | `"6007"`  |                                                                                                             |
+| global.HUGGINGFACEHUB_API_TOKEN | string | `""`      | Your own Hugging Face API token                                                                             |
+| DATAPREP_BACKEND                | string | `"REDIS"` | vector DB backend to use, one of "REDIS", "MILVUS", "QDRANT", "OPENSEARCH"                                  |
+| REDIS_HOST                      | string | `""`      | Redis service URL host, only valid for Redis, please see `values.yaml` for other Redis configuration        |
+| MILVUS_HOST                     | string | `""`      | Milvus service URL host, only valid for Milvus, please see `values.yaml` for other Milvus configuration     |
+| QDRANT_HOST                     | string | `""`      | Qdrant service URL host, only valid for Qdrant, please see `values.yaml` for other Qdrant configuration     |
+| OPENSEARCH_HOST                 | string | `""`      | OpenSearch URL host, only valid for OpenSearch, please see `values.yaml` for other OpenSearch configuration |
+| TEI_EMBEDDING_ENDPOINT          | string | `""`      |                                                                                                             |
+| global.monitoring               | bool   | `false`   | See ../../monitoring.md before enabling!                                                                    |
 
 ## Milvus support
 

helm-charts/common/data-prep/milvus-values.yaml

@@ -22,6 +22,8 @@ milvus:
       enabled: false
 redis-vector-db:
   enabled: false
+opensearch:
+  enabled: false
 tei:
   enabled: true
 

helm-charts/common/data-prep/opensearch-values.yaml

@@ -0,0 +1,28 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+DATAPREP_BACKEND: "OPENSEARCH"
+OPENSEARCH_HOST: "opensearch-cluster-master"
+OPENSEARCH_PORT: 9200
+OPENSEARCH_INITIAL_ADMIN_PASSWORD: "longPassWithNumCharCapLower1#"
+# SSL must be enabled unless you disable OpenSearch security, which is not recommended
+OPENSEARCH_SSL: true
+INDEX_NAME: "rag_opensearch"
+KEY_INDEX_NAME: "file-keys"
+TIMEOUT_SECONDS: 600
+SEARCH_BATCH_SIZE: 10
+
+tei:
+  enabled: true
+redis-vector-db:
+  enabled: false
+milvus:
+  enabled: false
+opensearch:
+  enabled: true
+  persistence:
+    enabled: false
+  singleNode: true
+  extraEnvs:
+    - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+      value: "longPassWithNumCharCapLower1#" # must match value above or data-prep cannot access OpenSearch

helm-charts/common/data-prep/qdrant-values.yaml

@@ -7,6 +7,8 @@ redis-vector-db:
   enabled: false
 milvus:
   enabled: false
+opensearch:
+  enabled: false
 qdrant:
   enabled: true
   tag: "v1.13.1"

helm-charts/common/data-prep/redis-values.yaml

@@ -7,3 +7,7 @@ redis-vector-db:
   enabled: true
 milvus:
   enabled: false
+qdrant:
+  enabled: false
+opensearch:
+  enabled: false

helm-charts/common/data-prep/templates/configmap.yaml

@@ -48,6 +48,24 @@ data:
   QDRANT_PORT: {{ .Values.QDRANT_PORT | quote }}
   COLLECTION_NAME: {{ .Values.COLLECTION_NAME | quote }}
   EMBED_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | default "" | quote }}
+  {{- else if eq "OPENSEARCH" .Values.DATAPREP_BACKEND }}
+  DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_OPENSEARCH"
+  {{- if .Values.OPENSEARCH_HOST }}
+  OPENSEARCH_HOST: {{ .Values.OPENSEARCH_HOST }}
+  {{- else }}
+  OPENSEARCH_HOST: "opensearch-cluster-master"
+  {{- end }}
+  OPENSEARCH_PORT: {{ .Values.OPENSEARCH_PORT | quote }}
+  {{- if lt (len .Values.OPENSEARCH_INITIAL_ADMIN_PASSWORD) 8 }}
+  {{ fail "OPENSEARCH_INITIAL_ADMIN_PASSWORD must be at least 8 characters" }}
+  {{- end }}
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ .Values.OPENSEARCH_INITIAL_ADMIN_PASSWORD | quote }}
+  OPENSEARCH_SSL: {{ .Values.OPENSEARCH_SSL | quote }}
+  INDEX_NAME: {{ .Values.INDEX_NAME | quote }}
+  KEY_INDEX_NAME: {{ .Values.KEY_INDEX_NAME | quote }}
+  TIMEOUT_SECONDS: {{ .Values.TIMEOUT_SECONDS | quote }}
+  SEARCH_BATCH_SIZE: {{ .Values.SEARCH_BATCH_SIZE | quote }}
+  EMBED_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | default "" | quote }}
   {{- else }}
   {{- cat "Invalid DATAPREP_BACKEND:" .Values.DATAPREP_BACKEND | fail }}
   {{- end }}

helm-charts/common/data-prep/templates/deployment.yaml

@@ -49,6 +49,9 @@ spec:
               {{- else if eq "QDRANT" .Values.DATAPREP_BACKEND }}
               TESTHOST=$(QDRANT_HOST);
               TESTPORT=$(QDRANT_PORT);
+              {{- else if eq "OPENSEARCH" .Values.DATAPREP_BACKEND }}
+              TESTHOST=$(OPENSEARCH_HOST);
+              TESTPORT=$(OPENSEARCH_PORT);
               {{- end }}
               wait_timeout={{ .Values.waitTimeout | default 1 }};
               total_timeout={{ .Values.retryTimeoutSeconds | default 60 }};

helm-charts/common/data-prep/values.yaml

@@ -33,6 +33,17 @@ SEARCH_BATCH_SIZE: 10
 # QDRANT_PORT: 6333
 # COLLECTION_NAME: "rag_qdrant"
 
+# Uncomment and set the following settings to use OpenSearch as a vector-DB backend
+# DATAPREP_BACKEND: "OPENSEARCH"
+# OPENSEARCH_HOST: ""
+# OPENSEARCH_PORT: 9200
+# OPENSEARCH_INITIAL_ADMIN_PASSWORD: ""
+# OPENSEARCH_SSL: true
+# INDEX_NAME: "rag_opensearch"
+# KEY_INDEX_NAME: "file-keys"
+# TIMEOUT_SECONDS: 600
+# SEARCH_BATCH_SIZE: 10
+
 # data-prep can do embedding locally or talk to remote TEI service to do embedding.
 # TEI_EMBEDDING_ENDPOINT takes precedence over LOCAL_EMBEDDING_MODEL.
 # TEI inference service URL, e.g. http://<service-name>:<port>
@@ -147,3 +158,5 @@ redis-vector-db:
   enabled: false
 qdrant:
   enabled: false
+opensearch:
+  enabled: false

helm-charts/common/retriever-usvc/Chart.yaml

@@ -28,3 +28,7 @@ dependencies:
   - name: commonlib
     version: 0-latest
     repository: "file://../commonlib"
+  - name: opensearch
+    version: 2.31.0
+    repository: https://opensearch-project.github.io/helm-charts
+    condition: opensearch.enabled

helm-charts/common/retriever-usvc/README.md

@@ -14,6 +14,8 @@ Helm chart for deploying OPEA retriever-usvc microservice.
 
 - Qdrant DB: please refer to [qdrant-helm](https://github.com/qdrant/qdrant-helm/tree/qdrant-1.13.1/charts/qdrant) for more information.
 
+- OpenSearch DB: please refer to [opensearch-helm](https://artifacthub.io/packages/helm/opensearch-project-helm-charts/opensearch) for more information.
+
 First, you need to install the `tei` helm chart and one of the vector DB service, i.e. `redis-vector-db` chart.
 
 After you've deployed dependency charts successfully, please run `kubectl get svc` to get the service endpoint URL respectively, i.e. `http://tei:80`, `redis://redis-vector-db:6379`.
@@ -23,7 +25,7 @@ To install `retriever-usvc` chart, run the following:
 ```console
 cd GenAIInfra/helm-charts/common/retriever-usvc
 helm dependency update
-export HFTOKEN="insert-your-huggingface-token-here"
+export HF_TOKEN="insert-your-huggingface-token-here"
 export TEI_EMBEDDING_ENDPOINT="http://tei"
 
 # Install retriever-usvc with Redis DB backend
@@ -40,6 +42,12 @@ helm install retriever-usvc . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPO
 # export RETRIEVER_BACKEND="QDRANT"
 # export DB_HOST="qdrant"
 # helm install retriever-usvc . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} --set global.HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} --set RETRIEVER_BACKEND=${RETRIEVER_BACKEND} --set QDRANT_HOST=${DB_HOST}
+
+# Install retriever-usvc with OpenSearch DB backend
+# export RETRIEVER_BACKEND="OPENSEARCH"
+# export DB_HOST="opensearch-cluster-master"
+# export OPENSEARCH_INITIAL_ADMIN_PASSWORD="insert-your-initial-admin-password"
+# helm install retriever-usvc . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} --set global.HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} --set RETRIEVER_BACKEND=${RETRIEVER_BACKEND} --set OPENSEARCH_HOST=${DB_HOST} --set OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} --set "opensearch.extraEnvs[0].name=OPENSEARCH_INITIAL_ADMIN_PASSWORD,opensearch.extraEnvs[0].value=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}" -f opensearch-values.yaml
 ```
 
 ## Verify
@@ -60,16 +68,17 @@ curl http://localhost:7000/v1/retrieval  \
 
 ## Values
 
-| Key                             | Type   | Default   | Description                                                                                             |
-| ------------------------------- | ------ | --------- | ------------------------------------------------------------------------------------------------------- |
-| global.HUGGINGFACEHUB_API_TOKEN | string | `""`      | Your own Hugging Face API token                                                                         |
-| service.port                    | string | `"7000"`  |                                                                                                         |
-| RETRIEVER_BACKEND               | string | `"REDIS"` | vector DB backend to use, one of "REDIS", "MILVUS", "QDRANT"                                            |
-| REDIS_HOST                      | string | `""`      | Redis service URL host, only valid for Redis, please see `values.yaml` for other Redis configuration    |
-| MILVUS_HOST                     | string | `""`      | Milvus service URL host, only valid for Milvus, please see `values.yaml` for other Milvus configuration |
-| QDRANT_HOST                     | string | `""`      | Qdrant service URL host, only valid for Qdrant, please see `values.yaml` for other Qdrant configuration |
-| TEI_EMBEDDING_ENDPOINT          | string | `""`      |                                                                                                         |
-| global.monitoring               | bool   | `false`   |                                                                                                         |
+| Key                             | Type   | Default   | Description                                                                                                         |
+| ------------------------------- | ------ | --------- | ------------------------------------------------------------------------------------------------------------------- |
+| global.HUGGINGFACEHUB_API_TOKEN | string | `""`      | Your own Hugging Face API token                                                                                     |
+| service.port                    | string | `"7000"`  |                                                                                                                     |
+| RETRIEVER_BACKEND               | string | `"REDIS"` | vector DB backend to use, one of "REDIS", "MILVUS", "QDRANT", "OPENSEARCH"                                          |
+| REDIS_HOST                      | string | `""`      | Redis service URL host, only valid for Redis, please see `values.yaml` for other Redis configuration                |
+| MILVUS_HOST                     | string | `""`      | Milvus service URL host, only valid for Milvus, please see `values.yaml` for other Milvus configuration             |
+| QDRANT_HOST                     | string | `""`      | Qdrant service URL host, only valid for Qdrant, please see `values.yaml` for other Qdrant configuration             |
+| OPENSEARCH_HOST                 | string | `""`      | OpenSearch service URL host, only valid for OpenSearch, please see `values.yaml` for other OpenSearch configuration |
+| TEI_EMBEDDING_ENDPOINT          | string | `""`      |                                                                                                                     |
+| global.monitoring               | bool   | `false`   |                                                                                                                     |
 
 ## Milvus support
 

helm-charts/common/retriever-usvc/milvus-values.yaml

@@ -22,6 +22,8 @@ milvus:
       enabled: false
 redis-vector-db:
   enabled: false
+opensearch:
+  enabled: false
 tei:
   enabled: true
 

helm-charts/common/retriever-usvc/opensearch-values.yaml

@@ -0,0 +1,27 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+RETRIEVER_BACKEND: "OPENSEARCH"
+OPENSEARCH_HOST: "opensearch-cluster-master"
+OPENSEARCH_PORT: 9200
+OPENSEARCH_INITIAL_ADMIN_PASSWORD: "longPassWithNumCharCapLower1#"
+# SSL must be enabled unless you disable OpenSearch security, which is not recommended
+OPENSEARCH_SSL: true
+OPENSEARCH_INDEX_NAME: "rag_opensearch"
+
+tei:
+  enabled: true
+redis-vector-db:
+  enabled: false
+milvus:
+  enabled: false
+qdrant:
+  enabled: false
+opensearch:
+  enabled: true
+  persistence:
+    enabled: false
+  singleNode: true
+  extraEnvs:
+    - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+      value: "longPassWithNumCharCapLower1#" # must match value above or retriever cannot access OpenSearch

helm-charts/common/retriever-usvc/qdrant-values.yaml

@@ -7,6 +7,8 @@ redis-vector-db:
   enabled: false
 milvus:
   enabled: false
+opensearch:
+  enabled: false
 qdrant:
   enabled: true
   tag: "v1.13.1"