feat: Remove unused configs to make benchmarks reproducible

qdrant · Jan 12, 2024 · 405d87c · 405d87c
1 parent c50cf9f
commit 405d87c
Show file tree

Hide file tree

Showing 14 changed files with 25 additions and 178 deletions.
diff --git a/engine/servers/qdrant-billion-scale/docker-compose.yaml b/engine/servers/qdrant-billion-scale/docker-compose.yaml
@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   qdrant_bench:
-    image: qdrant/qdrant:dev
+    image: qdrant/qdrant:v1.7.3
     network_mode: host
     volumes:
       - ./storage:/qdrant/storage

diff --git a/engine/servers/qdrant-cluster-mode/docker-compose.yaml b/engine/servers/qdrant-cluster-mode/docker-compose.yaml
@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   qdrant_node_1:
-    image: qdrant/qdrant:v1.5.1
+    image: qdrant/qdrant:v1.7.3
     environment:
       - QDRANT__SERVICE__GRPC_PORT=6334
       - QDRANT__CLUSTER__ENABLED=true
@@ -18,7 +18,7 @@ services:
 
 
   qdrant_node_follower:
-    image: qdrant/qdrant:v1.5.1
+    image: qdrant/qdrant:v1.7.3
     environment:
       - QDRANT__SERVICE__GRPC_PORT=6334
       - QDRANT__CLUSTER__ENABLED=true
@@ -36,7 +36,7 @@ services:
 
 
   qdrant_node_follower_2:
-    image: qdrant/qdrant:v1.5.1
+    image: qdrant/qdrant:v1.7.3
     environment:
       - QDRANT__SERVICE__GRPC_PORT=6334
       - QDRANT__CLUSTER__ENABLED=true

diff --git a/engine/servers/qdrant-limit-ram/docker-compose.yaml b/engine/servers/qdrant-limit-ram/docker-compose.yaml
@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   qdrant_bench:
-    image: qdrant/qdrant:v1.5.1
+    image: qdrant/qdrant:v1.7.3
     network_mode: host
     logging:
       driver: "json-file"

diff --git a/engine/servers/qdrant-single-node/docker-compose-limit-cpu.yaml.yml b/engine/servers/qdrant-single-node/docker-compose-limit-cpu.yaml.yml
@@ -0,0 +1,16 @@
+version: '3.7'
+
+services:
+  qdrant_bench:
+    image: qdrant/qdrant:${QDRANT_VERSION:-v1.7.3}
+    network_mode: host
+    logging:
+      driver: "json-file"
+      options:
+        max-file: 1
+        max-size: 10m
+    deploy:
+      resources:
+        limits:
+          # memory: 25Gb
+          cpus: '${QDRANT_CPU_LIMIT:-10000}' # this forces you to set a limit while running
diff --git a/engine/servers/qdrant-single-node/docker-compose.yaml b/engine/servers/qdrant-single-node/docker-compose.yaml
@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   qdrant_bench:
-    image: qdrant/qdrant:${QDRANT_VERSION:-v1.5.1}
+    image: qdrant/qdrant:v1.7.3
     network_mode: host
     logging:
       driver: "json-file"

diff --git a/engine/servers/weaviate-single-node/docker-compose.yaml b/engine/servers/weaviate-single-node/docker-compose.yaml
@@ -23,7 +23,7 @@ services:
       DEFAULT_VECTORIZER_MODULE: 'none'
       ENABLE_MODULES: ''
       CLUSTER_HOSTNAME: 'node1'
-      GOMEMLIMIT: 25GiB
+      GOMEMLIMIT: 25GiB # https://weaviate.io/blog/gomemlimit-a-game-changer-for-high-memory-applications
       GOGC: 50
     deploy:
       resources:

diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json
@@ -8,9 +8,6 @@
     "collection_params": { "index_options": { "m": 16, "ef_construction": 100 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }
@@ -24,9 +21,6 @@
     "collection_params": { "index_options": { "m": 16, "ef_construction": 128 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }
@@ -40,9 +34,6 @@
     "collection_params": { "index_options": { "m": 32, "ef_construction": 128 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }
@@ -56,9 +47,6 @@
     "collection_params": { "index_options": { "m": 32, "ef_construction": 256 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }
@@ -72,9 +60,6 @@
     "collection_params": { "index_options": { "m": 32, "ef_construction": 512 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }
@@ -88,9 +73,6 @@
     "collection_params": { "index_options": { "m": 64, "ef_construction": 256 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }
@@ -104,9 +86,6 @@
     "collection_params": { "index_options": { "m": 64, "ef_construction": 512 } },
     "search_params": [
       { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
-      { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
-      { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
       { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
     ],
     "upload_params": { "parallel": 16 }

diff --git a/experiments/configurations/milvus-single-node.json b/experiments/configurations/milvus-single-node.json
@@ -6,9 +6,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 100, "M": 16 } }
@@ -20,9 +17,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 128, "M": 16 } }
@@ -34,9 +28,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 128, "M": 32 } }
@@ -48,9 +39,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 256, "M": 32 } }
@@ -62,9 +50,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 512, "M": 32 } }
@@ -76,9 +61,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 256, "M": 64 } }
@@ -90,9 +72,6 @@
     "collection_params": {},
     "search_params": [
       { "parallel": 1, "params": { "ef": 128 } }, { "parallel": 1, "params": { "ef": 256 } }, { "parallel": 1, "params": { "ef": 512 } },
-      { "parallel": 2, "params": { "ef": 128 } }, { "parallel": 2, "params": { "ef": 256 } }, { "parallel": 2, "params": { "ef": 512 } },
-      { "parallel": 4, "params": { "ef": 128 } }, { "parallel": 4, "params": { "ef": 256 } }, { "parallel": 4, "params": { "ef": 512 } },
-      { "parallel": 8, "params": { "ef": 128 } }, { "parallel": 8, "params": { "ef": 256 } }, { "parallel": 8, "params": { "ef": 512 } },
       { "parallel": 100, "params": { "ef": 128 } }, { "parallel": 100, "params": { "ef": 256 } }, { "parallel": 100, "params": { "ef": 512 } }
     ],
     "upload_params": { "parallel": 16, "index_params": { "efConstruction": 512, "M": 64 } }