feat: Use ghcr for ci benchmarks and store commit id (#93)

* feat: Use ghcr for ci benchmarks and store commit id * fix: Add missing qdrant_collect_stats.sh
qdrant · Jan 18, 2024 · a007179 · a007179
1 parent 4224a27
commit a007179
Show file tree

Hide file tree

Showing 8 changed files with 38 additions and 14 deletions.
diff --git a/.github/workflows/continuous-benchmark.yaml b/.github/workflows/continuous-benchmark.yaml
@@ -23,7 +23,7 @@ jobs:
           export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
 
           # Benchmark the dev branch:
-          export QDRANT_VERSION=dev
+          export QDRANT_VERSION=ghcr/dev
           bash -x tools/run_ci.sh
 
           # Benchmark the master branch:

diff --git a/.github/workflows/manual-benchmark.yaml b/.github/workflows/manual-benchmark.yaml
@@ -5,8 +5,8 @@ on:
   workflow_dispatch:
     inputs:
         qdrant_version:
-            description: "Version of qdrant to benchmark (tags/v1.6.1, <commit-id>, my-branch, docker/v1.5.1)"
-            default: dev
+            description: "Version of qdrant to benchmark (tags/v1.6.1, <commit-id>, my-branch, docker/v1.5.1, ghcr/dev)"
+            default: ghcr/dev
         dataset:
             description: "Dataset to benchmark"
             default: laion-small-clip

diff --git a/engine/servers/qdrant-single-node/docker-compose.yaml b/engine/servers/qdrant-single-node/docker-compose.yaml
@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   qdrant_bench:
-    image: qdrant/qdrant:v1.7.3
+    image: ${CONTAINER_REGISTRY:-docker.io}/qdrant/qdrant:v1.7.3
     network_mode: host
     logging:
       driver: "json-file"

diff --git a/tools/qdrant_mem_usage.sh → tools/qdrant_collect_stats.sh b/tools/qdrant_mem_usage.sh → tools/qdrant_collect_stats.sh
@@ -20,4 +20,8 @@ MEMORY_USAGE=$(ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "grep VmRSS /proc
 
 CURRENT_DATE=$(date +%Y-%m-%d-%H-%M-%S)
 
-echo $MEMORY_USAGE > results/memory-usage-${CURRENT_DATE}.txt
+echo "$MEMORY_USAGE" > results/memory-usage-"${CURRENT_DATE}".txt
+
+ROOT_API_RESPONSE=$(ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "curl -s http://localhost:6333/")
+
+echo "$ROOT_API_RESPONSE" > results/root-api-"${CURRENT_DATE}".json
diff --git a/tools/run_ci.sh b/tools/run_ci.sh
@@ -37,5 +37,6 @@ bash -x "${SCRIPT_PATH}/run_remote_benchmark.sh"
 export SEARCH_RESULTS_FILE=$(ls -t results/*-search-*.json | head -n 1)
 export UPLOAD_RESULTS_FILE=$(ls -t results/*-upload-*.json | head -n 1)
 export MEMORY_USAGE_FILE=$(ls -t results/memory-usage-*.txt | head -n 1)
+export ROOT_API_RESPONSE_FILE=$(ls -t results/root-api-*.json | head -n 1)
 
 bash -x "${SCRIPT_PATH}/upload_results_postgres.sh"
diff --git a/tools/run_remote_benchmark.sh b/tools/run_remote_benchmark.sh
@@ -40,4 +40,4 @@ bash -x "${SCRIPT_PATH}/run_server_container.sh" "$SERVER_CONTAINER_NAME"
 
 bash -x "${SCRIPT_PATH}/run_client_script.sh"
 
-bash -x "${SCRIPT_PATH}/qdrant_mem_usage.sh" "$SERVER_CONTAINER_NAME"
+bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"
diff --git a/tools/run_server_container.sh b/tools/run_server_container.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-# Examples: qdrant-single-node, qdrant-single-node-rps
+# Examples: qdrant-single-node
 CONTAINER_NAME=$1
 
 CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
@@ -20,10 +20,20 @@ IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_
 
 bash -x "${SCRIPT_PATH}/sync_servers.sh" "root@$IP_OF_THE_SERVER"
 
-# if version is dev or if starts with "docker", run in docker
-if [ "${QDRANT_VERSION}" == "dev" ] || [[ ${QDRANT_VERSION} == docker/* ]]; then
-    QDRANT_VERSION=${QDRANT_VERSION#docker/}
-    DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; docker compose down ; pkill qdrant ; docker rmi qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d"
+# if version is dev or if starts with "docker" or "ghcr", use container
+if [[ ${QDRANT_VERSION} == docker/* ]] || [[ ${QDRANT_VERSION} == ghcr/* ]]; then
+
+    if [[ ${QDRANT_VERSION} == docker/* ]]; then
+        # pull from docker hub
+        QDRANT_VERSION=${QDRANT_VERSION#docker/}
+        CONTAINER_REGISTRY='docker.io'
+    elif [[ ${QDRANT_VERSION} == ghcr/* ]]; then
+        # pull from github container registry
+        QDRANT_VERSION=${QDRANT_VERSION#ghcr/}
+        CONTAINER_REGISTRY='ghcr.io'
+    fi
+
+    DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; docker compose down ; pkill qdrant ; docker rmi ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d"
     ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/vector-db-benchmark/engine/servers/${CONTAINER_NAME} ; $DOCKER_COMPOSE"
 else
     # else run natively in the server

diff --git a/tools/upload_results_postgres.sh b/tools/upload_results_postgres.sh
@@ -7,7 +7,8 @@
 # create table benchmark (
 # 	id SERIAL PRIMARY key,
 # 	engine VARCHAR(255),
-# 	branch VARCCHAR(255),
+# 	branch VARCHAR(255),
+#   commit CHAR(40),
 # 	dataset VARCHAR(255),
 # 	measure_timestamp TIMESTAMP,
 # 	upload_time real,
@@ -22,6 +23,7 @@
 SEARCH_RESULTS_FILE=${SEARCH_RESULTS_FILE:-""}
 UPLOAD_RESULTS_FILE=${UPLOAD_RESULTS_FILE:-""}
 MEMORY_USAGE_FILE=${MEMORY_USAGE_FILE:-""}
+ROOT_API_RESPONSE_FILE=${ROOT_API_RESPONSE_FILE:-""}
 POSTGRES_TABLE=${POSTGRES_TABLE:-"benchmark"}
 
 QDRANT_VERSION=${QDRANT_VERSION:-"dev"}
@@ -44,6 +46,11 @@ if [[ -z "$MEMORY_USAGE_FILE" ]]; then
   exit 1
 fi
 
+if [[ -z "$ROOT_API_RESPONSE_FILE" ]]; then
+  echo "ROOT_API_RESPONSE_FILE is not set"
+  exit 1
+fi
+
 RPS=$(jq -r '.results.rps' "$SEARCH_RESULTS_FILE")
 MEAN_PRECISIONS=$(jq -r '.results.mean_precisions' "$SEARCH_RESULTS_FILE")
 P95_TIME=$(jq -r '.results.p95_time' "$SEARCH_RESULTS_FILE")
@@ -54,11 +61,13 @@ INDEXING_TIME=$(jq -r '.results.total_time' "$UPLOAD_RESULTS_FILE")
 
 MEMORY_USAGE=$(cat "$MEMORY_USAGE_FILE")
 
+QDRANT_COMMIT=$(jq -r '.commit' "$ROOT_API_RESPONSE_FILE")
+
 MEASURE_TIMESTAMP=${MEASURE_TIMESTAMP:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")}
 
 
 docker run --rm jbergknoff/postgresql-client "postgresql://qdrant:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/postgres" -c "
-INSERT INTO ${POSTGRES_TABLE} (engine, branch, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
-VALUES ('qdrant-ci', '${QDRANT_VERSION}', '${DATASETS}', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE});
+INSERT INTO ${POSTGRES_TABLE} (engine, branch, commit, dataset, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
+VALUES ('qdrant-ci', '${QDRANT_VERSION}', '${QDRANT_COMMIT}', '${DATASETS}', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE});
 "
Original file line number	Diff line number	Diff line change
Expand Up		@@ -40,4 +40,4 @@ bash -x "${SCRIPT_PATH}/run_server_container.sh" "$SERVER_CONTAINER_NAME"

		bash -x "${SCRIPT_PATH}/run_client_script.sh"

		bash -x "${SCRIPT_PATH}/qdrant_mem_usage.sh" "$SERVER_CONTAINER_NAME"
		bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"