Continuous Benchmark #2138
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Benchmark | |
| on: | |
| repository_dispatch: | |
| workflow_dispatch: | |
| schedule: | |
| # Run every 4 hours | |
| - cron: "0 */4 * * *" | |
| jobs: | |
| runBenchmark: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| bash -x tools/setup_ci.sh | |
| declare -A DATASET_TO_ENGINE | |
| DATASET_TO_ENGINE["laion-small-clip"]="qdrant-continuous-benchmark" | |
| DATASET_TO_ENGINE["msmarco-sparse-100K"]="qdrant-sparse-vector" | |
| DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark" | |
| DATASET_TO_ENGINE["dbpedia-openai-100K-1536-angular"]="qdrant-bq-continuous-benchmark" | |
| set +e | |
| for dataset in "${!DATASET_TO_ENGINE[@]}"; do | |
| export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]} | |
| export DATASETS=$dataset | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| timeout 30m bash -x tools/run_ci.sh | |
| done | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| runTenantsBenchmark: | |
| runs-on: ubuntu-latest | |
| needs: runBenchmark | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| bash -x tools/setup_ci.sh | |
| set +e | |
| # Benchmark filtered search by tenants with mem limitation | |
| export ENGINE_NAME="qdrant-all-on-disk-scalar-q" | |
| export DATASETS="random-768-100-tenants" | |
| export BENCHMARK_STRATEGY="tenants" | |
| export CONTAINER_MEM_LIMIT=160mb | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| timeout 30m bash -x tools/run_ci.sh | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runTenantsBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runTenantsBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| runLoadTimeBenchmark: | |
| runs-on: ubuntu-latest | |
| needs: runBenchmark | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| export SERVER_NAME="benchmark-server-3" | |
| bash -x tools/setup_ci.sh | |
| set +e | |
| # Benchmark collection load time | |
| export BENCHMARK_STRATEGY="collection-reload" | |
| declare -A DATASET_TO_ENGINE | |
| declare -A DATASET_TO_URL | |
| DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot" | |
| DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot" | |
| DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot" | |
| DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot" | |
| export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads" | |
| DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot" | |
| DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot" | |
| DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot" | |
| DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot" | |
| set +e | |
| for dataset in "${!DATASET_TO_ENGINE[@]}"; do | |
| export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]} | |
| export DATASETS=$dataset | |
| export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]} | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| timeout 30m bash -x tools/run_ci.sh | |
| done | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| runParallelBenchmark: | |
| runs-on: ubuntu-latest | |
| needs: [ runLoadTimeBenchmark, runTenantsBenchmark ] | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| bash -x tools/setup_ci.sh | |
| set +e | |
| # Benchmark parallel search&upload | |
| export ENGINE_NAME="qdrant-continuous-benchmark" | |
| export DATASETS="laion-small-clip" | |
| export BENCHMARK_STRATEGY="parallel" | |
| export POSTGRES_TABLE="benchmark_parallel_search_upload" | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| timeout 30m bash -x tools/run_ci.sh | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runTenantsBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runTenantsBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |