better logging

qdrant · Mar 14, 2024 · c334232 · c334232
1 parent d37d7ff
commit c334232
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 4 deletions.
diff --git a/engine/base_client/client.py b/engine/base_client/client.py
@@ -1,4 +1,5 @@
 import json
+import os
 from datetime import datetime
 from pathlib import Path
 from typing import List
@@ -12,6 +13,8 @@
 RESULTS_DIR = ROOT_DIR / "results"
 RESULTS_DIR.mkdir(exist_ok=True)
 
+DETAILED_RESULTS = bool(int(os.getenv("DETAILED_RESULTS", False)))
+
 
 class BaseClient:
     def __init__(
@@ -49,7 +52,11 @@ def save_upload_results(
         experiments_file = f"{self.name}-{dataset_name}-upload-{timestamp}.json"
         with open(RESULTS_DIR / experiments_file, "w") as out:
             upload_stats = {
-                "params": upload_params,
+                "params": {
+                    "engine": self.name,
+                    "dataset": dataset_name,
+                    **upload_params
+                },
                 "results": results,
             }
             out.write(json.dumps(upload_stats, indent=2))
@@ -84,6 +91,11 @@ def run_experiment(
             upload_stats = self.uploader.upload(
                 distance=dataset.config.distance, records=reader.read_data()
             )
+
+            if not DETAILED_RESULTS:
+                # Remove verbose stats from upload results
+                upload_stats.pop("latencies", None)
+
             self.save_upload_results(
                 dataset.config.name,
                 upload_stats,
@@ -113,6 +125,11 @@ def run_experiment(
                 search_stats = searcher.search_all(
                     dataset.config.distance, reader.read_queries()
                 )
+                if not DETAILED_RESULTS:
+                    # Remove verbose stats from search results
+                    search_stats.pop("latencies", None)
+                    search_stats.pop("precisions", None)
+
                 self.save_search_results(
                     dataset.config.name, search_stats, search_id, search_params
                 )

diff --git a/experiments/configurations/qdrant-vs-weaviate.json b/experiments/configurations/qdrant-vs-weaviate.json
@@ -1,12 +1,35 @@
 [
   {
-    "name": "qdrant-proposed-config",
+    "name": "proposed-config-qdrant-latency",
     "engine": "qdrant",
     "connection_params": { "timeout": 60 },
     "collection_params": {
       "optimizers_config": { "memmap_threshold":10000000 },
       "hnsw_config": { "m": 32, "ef_construct": 256 }
     },
+    "search_params": [
+     { "parallel": 1, "vectorIndexConfig": { "ef": 16} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 32} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 64} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 128} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 256} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 512} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 768} }
+    ],
+    "upload_params": { "parallel": 8, "batch_size": 1024 }
+  },
+  {
+    "name": "proposed-config-qdrant-rps",
+    "engine": "qdrant",
+    "connection_params": { "timeout": 60 },
+    "collection_params": {
+      "optimizers_config": {
+        "memmap_threshold":10000000,
+        "max_segment_size": 1000000,
+        "default_segment_number":2
+      },
+      "hnsw_config": { "m": 32, "ef_construct": 256 }
+    },
     "search_params": [
       { "parallel": 100, "search_params": { "hnsw_ef": 16 } },
       { "parallel": 100, "search_params": { "hnsw_ef": 32 } },
@@ -19,7 +42,7 @@
     "upload_params": { "parallel": 8, "batch_size": 1024 }
   },
   {
-   "name": "weaviate-proposed-config",
+   "name": "proposed-config-weaviate",
    "engine": "weaviate",
    "connection_params": {
      "timeout_config": 60
@@ -31,13 +54,21 @@
        }
    },
    "search_params": [
+
      { "parallel": 100, "vectorIndexConfig": { "ef": 16} },
      { "parallel": 100, "vectorIndexConfig": { "ef": 32} },
      { "parallel": 100, "vectorIndexConfig": { "ef": 64} },
      { "parallel": 100, "vectorIndexConfig": { "ef": 128} },
      { "parallel": 100, "vectorIndexConfig": { "ef": 256} },
      { "parallel": 100, "vectorIndexConfig": { "ef": 512} },
-     { "parallel": 100, "vectorIndexConfig": { "ef": 768} }
+     { "parallel": 100, "vectorIndexConfig": { "ef": 768} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 16} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 32} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 64} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 128} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 256} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 512} },
+     { "parallel": 1, "vectorIndexConfig": { "ef": 768} }
    ],
    "upload_params": { "parallel": 8, "batch_size": 1024 }
   }