-
Notifications
You must be signed in to change notification settings - Fork 92
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #218 from qdrant/ci/reduce-snapshots-bench-runs
Only run runLoadTimeBenchmark once per day
- Loading branch information
Showing
2 changed files
with
126 additions
and
106 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
name: Continuous Benchmark 2 | ||
|
||
on: | ||
repository_dispatch: | ||
workflow_dispatch: | ||
schedule: | ||
# Run every day at midnight | ||
- cron: "0 0 * * *" | ||
|
||
# Restrict to only running this workflow one at a time. | ||
# Any new runs will be queued until the previous run is complete. | ||
# Any existing pending runs will be cancelled and replaced with current run. | ||
concurrency: | ||
group: continuous-benchmark | ||
|
||
jobs: | ||
# Schedule this benchmark to run once a day for the sake of saving on S3 costs. | ||
runLoadTimeBenchmark: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: webfactory/[email protected] | ||
with: | ||
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | ||
- name: Benches | ||
id: benches | ||
run: | | ||
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | ||
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | ||
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | ||
export SERVER_NAME="benchmark-server-3" | ||
bash -x tools/setup_ci.sh | ||
set +e | ||
# Benchmark collection load time | ||
export BENCHMARK_STRATEGY="collection-reload" | ||
declare -A DATASET_TO_ENGINE | ||
declare -A DATASET_TO_URL | ||
DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot" | ||
DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot" | ||
DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot" | ||
DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot" | ||
export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads" | ||
DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot" | ||
DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot" | ||
DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot" | ||
DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot" | ||
set +e | ||
for dataset in "${!DATASET_TO_ENGINE[@]}"; do | ||
export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]} | ||
export DATASETS=$dataset | ||
export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]} | ||
# Benchmark the dev branch: | ||
export QDRANT_VERSION=ghcr/dev | ||
timeout 30m bash -x tools/run_ci.sh | ||
# Benchmark the master branch: | ||
export QDRANT_VERSION=docker/master | ||
timeout 30m bash -x tools/run_ci.sh | ||
done | ||
set -e | ||
- name: Fail job if any of the benches failed | ||
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | ||
run: exit 1 | ||
- name: Send Notification | ||
if: failure() || cancelled() | ||
uses: slackapi/[email protected] | ||
with: | ||
payload: | | ||
{ | ||
"text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}", | ||
"blocks": [ | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | ||
} | ||
} | ||
] | ||
} | ||
env: | ||
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | ||
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,6 +7,12 @@ on: | |
# Run every 4 hours | ||
- cron: "0 */4 * * *" | ||
|
||
# Restrict to only running this workflow one at a time. | ||
# Any new runs will be queued until the previous run is complete. | ||
# Any existing pending runs will be cancelled and replaced with current run. | ||
concurrency: | ||
group: continuous-benchmark | ||
|
||
jobs: | ||
runBenchmark: | ||
runs-on: ubuntu-latest | ||
|
@@ -182,114 +188,9 @@ jobs: | |
env: | ||
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | ||
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | ||
runLoadTimeBenchmark: | ||
runs-on: ubuntu-latest | ||
needs: runBenchmark | ||
if: ${{ always() }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: webfactory/[email protected] | ||
with: | ||
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | ||
- name: Benches | ||
id: benches | ||
run: | | ||
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | ||
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | ||
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | ||
export SERVER_NAME="benchmark-server-3" | ||
bash -x tools/setup_ci.sh | ||
set +e | ||
# Benchmark collection load time | ||
export BENCHMARK_STRATEGY="collection-reload" | ||
declare -A DATASET_TO_ENGINE | ||
declare -A DATASET_TO_URL | ||
DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot" | ||
DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot" | ||
DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot" | ||
DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot" | ||
export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads" | ||
DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot" | ||
DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot" | ||
DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot" | ||
DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot" | ||
set +e | ||
for dataset in "${!DATASET_TO_ENGINE[@]}"; do | ||
export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]} | ||
export DATASETS=$dataset | ||
export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]} | ||
# Benchmark the dev branch: | ||
export QDRANT_VERSION=ghcr/dev | ||
timeout 30m bash -x tools/run_ci.sh | ||
# Benchmark the master branch: | ||
export QDRANT_VERSION=docker/master | ||
timeout 30m bash -x tools/run_ci.sh | ||
done | ||
set -e | ||
- name: Fail job if any of the benches failed | ||
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | ||
run: exit 1 | ||
- name: Send Notification | ||
if: failure() || cancelled() | ||
uses: slackapi/[email protected] | ||
with: | ||
payload: | | ||
{ | ||
"text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}", | ||
"blocks": [ | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | ||
} | ||
}, | ||
{ | ||
"type": "section", | ||
"text": { | ||
"type": "mrkdwn", | ||
"text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | ||
} | ||
} | ||
] | ||
} | ||
env: | ||
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | ||
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | ||
|
||
runParallelBenchmark: | ||
runs-on: ubuntu-latest | ||
needs: [ runLoadTimeBenchmark, runTenantsBenchmark ] | ||
needs: runTenantsBenchmark | ||
if: ${{ always() }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
|