Continuous Benchmark #2896
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Benchmark | |
| on: | |
| repository_dispatch: | |
| workflow_dispatch: | |
| inputs: | |
| with_payload: | |
| description: 'Flag that controls whether to search with or without payload (false or true)' | |
| default: false | |
| schedule: | |
| # Run every 4 hours | |
| - cron: "0 */4 * * *" | |
| # Restrict to only running this workflow one at a time. | |
| # Any new runs will be queued until the previous run is complete. | |
| # Any existing pending runs will be cancelled and replaced with current run. | |
| concurrency: | |
| group: continuous-benchmark | |
| jobs: | |
| runBenchmark: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/ssh-agent@v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| export WITH_PAYLOAD=${{ inputs.with_payload }} | |
| export SUFFIX=$([ "${WITH_PAYLOAD}" = "true" ] && echo "-with-payload" || echo "") | |
| bash -x tools/setup_ci.sh | |
| declare -A DATASET_TO_ENGINE | |
| DATASET_TO_ENGINE["laion-small-clip"]="qdrant-continuous-benchmark${SUFFIX}" | |
| DATASET_TO_ENGINE["msmarco-sparse-100K"]="qdrant-sparse-vector${SUFFIX}" | |
| DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark${SUFFIX}" | |
| DATASET_TO_ENGINE["dbpedia-openai-100K-1536-angular"]="qdrant-bq-continuous-benchmark${SUFFIX}" | |
| set +e | |
| for dataset in "${!DATASET_TO_ENGINE[@]}"; do | |
| export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]} | |
| export DATASETS=$dataset | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| export QDRANT__FEATURE_FLAGS__ALL=true | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| export QDRANT__FEATURE_FLAGS__ALL=false | |
| timeout 30m bash -x tools/run_ci.sh | |
| done | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| runTenantsBenchmark: | |
| runs-on: ubuntu-latest | |
| needs: runBenchmark | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/ssh-agent@v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| export WITH_PAYLOAD=${{ inputs.with_payload }} | |
| export SUFFIX=$([ "${WITH_PAYLOAD}" = "true" ] && echo "-with-payload" || echo "") | |
| bash -x tools/setup_ci.sh | |
| set +e | |
| # Benchmark filtered search by tenants with mem limitation | |
| export ENGINE_NAME="qdrant-all-on-disk-scalar-q${SUFFIX}" | |
| export DATASETS="random-768-100-tenants" | |
| export BENCHMARK_STRATEGY="tenants" | |
| export CONTAINER_MEM_LIMIT=160mb | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| export QDRANT__FEATURE_FLAGS__ALL=true | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| export QDRANT__FEATURE_FLAGS__ALL=false | |
| timeout 30m bash -x tools/run_ci.sh | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runTenantsBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runTenantsBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| runParallelBenchmark: | |
| runs-on: ubuntu-latest | |
| needs: runTenantsBenchmark | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: webfactory/ssh-agent@v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Benches | |
| id: benches | |
| run: | | |
| export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }} | |
| export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} | |
| export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }} | |
| export WITH_PAYLOAD=${{ inputs.with_payload }} | |
| export SUFFIX=$([ "${WITH_PAYLOAD}" = "true" ] && echo "-with-payload" || echo "") | |
| bash -x tools/setup_ci.sh | |
| set +e | |
| # Benchmark parallel search&upload | |
| export ENGINE_NAME="qdrant-continuous-benchmark${SUFFIX}" | |
| export DATASETS="laion-small-clip" | |
| export BENCHMARK_STRATEGY="parallel" | |
| export POSTGRES_TABLE="benchmark_parallel_search_upload" | |
| # Benchmark the dev branch: | |
| export QDRANT_VERSION=ghcr/dev | |
| export QDRANT__FEATURE_FLAGS__ALL=true | |
| timeout 30m bash -x tools/run_ci.sh | |
| # Benchmark the master branch: | |
| export QDRANT_VERSION=docker/master | |
| export QDRANT__FEATURE_FLAGS__ALL=false | |
| timeout 30m bash -x tools/run_ci.sh | |
| set -e | |
| - name: Fail job if any of the benches failed | |
| if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout' | |
| run: exit 1 | |
| - name: Send Notification | |
| if: failure() || cancelled() | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload: | | |
| { | |
| "text": "CI benchmarks (runParallelBenchmark) run status: ${{ job.status }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "CI benchmarks (runParallelBenchmark) failed because of *${{ steps.benches.outputs.failed }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Engine: *${{ steps.benches.outputs.engine_name }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Dataset: *${{ steps.benches.outputs.dataset }}*." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |