Skip to content

Benchmarks

Benchmarks #268

Workflow file for this run

name: Benchmarks
on:
workflow_dispatch:
inputs:
change-type:
description: How should changes be made after the benchmarks are completed?
required: true
default: ignore
type: choice
options:
- ignore
- commit
- pull_request
instance_type:
description: Instance type to provision for benchmarking.
required: true
default: 'c6i.metal'
instance_disk_size:
description: Disk size (in GB) for the instance.
required: true
default: 100
concurrency:
group: benchmarks
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
TERM: xterm-256color
jobs:
provision-runner:
runs-on: ubuntu-latest
outputs:
available_region: ${{ steps.get-region.outputs.available_region }}
steps:
- name: AWS Login
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Generate Runner Registration Token
id: get-runner-token
run: |
RESPONSE=$(curl -L -X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.RUNNER_TOKEN }}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ github.repository }}/actions/runners/registration-token)
TOKEN=$(echo "$RESPONSE" | jq -r .token)
echo "REGISTER_TOKEN=$TOKEN" >> $GITHUB_ENV
echo "::add-mask::$TOKEN"
- name: Checkout
uses: actions/checkout@v4
- name: Check Capacity
id: get-region
run: |
regions=(us-east-1 us-east-2 us-west-1 us-west-2 ap-south-1 ap-southeast-1 ap-southeast-2 ap-southeast-3 ap-northeast-1 ap-northeast-2 ap-northeast-3 ca-central-1 eu-central-1 eu-west-1 eu-west-2 eu-west-3 eu-north-1 sa-east-1)
instance_type="${{ inputs.instance_type }}"
available_region=""
for region in "${regions[@]}"
do
available_region=$(aws ec2 describe-instance-type-offerings \
--location-type availability-zone \
--filters Name=instance-type,Values=$instance_type \
--region $region \
--query "InstanceTypeOfferings[?InstanceType=='$instance_type'].InstanceType" \
--output text)
if [ -n "$available_region" ]; then
echo "AVAILABLE_REGION=$region" >> $GITHUB_ENV
echo "available_region=$region" >> $GITHUB_OUTPUT
break
fi
done
- name: Check Region
run: |
if [ -z "${{ env.AVAILABLE_REGION }}" ]; then
echo "No available regions for instance type ${{ inputs.instance_type }}"
exit 1
fi
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
- name: Terraform Apply
timeout-minutes: 30
run: |
cd terraform/bench-runner
terraform init
terraform apply -auto-approve -var="aws_region=${{ env.AVAILABLE_REGION }}" -var="instance_type=${{ inputs.instance_type }}" -var="instance_disk_size=${{ inputs.instance_disk_size }}" -var="registration_token=${{ env.REGISTER_TOKEN }}" -var="github_run_id=${{ github.run_id }}"
- name: Notify on Failure
if: failure()
uses: appleboy/telegram-action@master
with:
to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }}
token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }}
format: markdown
disable_web_page_preview: true
message: |
*Status*: 🔥
*Problem*: Issue with server provisioning - bench-runner-${{ github.run_id }}
*Details*: Check [Benchmark Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
benchmarks:
needs: provision-runner
runs-on: [bench]
permissions:
contents: write
pull-requests: write
env:
RUSTUP_HOME: /tmp/rustup_home
steps:
- name: "ACTIONS: Checkout"
uses: actions/checkout@v4
- name: "Install: Rust toolchain"
uses: dsherret/rust-toolchain-file@v1
- name: "Install: Build deps"
run: |
sudo apt update
sudo apt install -y git clang curl libssl-dev llvm libudev-dev cmake wabt protobuf-compiler wget bzip2
curl -o wasm-opt-linux-x64.tar.gz -L `curl -s https://api.github.com/repos/WebAssembly/binaryen/releases/latest | jq -r '.assets[] | select(.name | contains("x86_64-linux")) | .browser_download_url' |grep -v sha256`
tar xzf wasm-opt-linux-x64.tar.gz && sudo mv binaryen-version_*/bin/* /usr/local/bin/
cargo install cargo-script
wasm-opt --version
- name: Run all benchmarks
run: |
INSTANCE_TYPE="${{ inputs.instance_type }}"
INSTANCE_TYPE=${INSTANCE_TYPE} ./scripts/benchmarking/run_all_benchmarks.sh -c vara-dev -m
cp ./scripts/benchmarking/weights-output/* runtime/vara/src/weights
# apply weight adjustments for baremetal runner to `runtime/vara/src/weights/pallet_gear.rs`
if [ "$INSTANCE_TYPE" = "c6i.metal" ]; then
echo "Applying baremetal weight patch"
./scripts/benchmarking/apply_baremetal_weight_patch.rs -i runtime/vara/src/weights/pallet_gear.rs -o runtime/vara/src/weights/pallet_gear_patched.rs
mv runtime/vara/src/weights/pallet_gear_patched.rs runtime/vara/src/weights/pallet_gear.rs
fi
# apply some patches for `pallets/gear/src/weights.rs`
cp runtime/vara/src/weights/pallet_gear.rs pallets/gear/src/weights.rs
sed -i -E 's/\w+::WeightInfo for SubstrateWeight/WeightInfo for SubstrateWeight/' pallets/gear/src/weights.rs
# apply some patches for `pallets/gear-builtin/src/weights.rs`
cp runtime/vara/src/weights/pallet_gear_builtin.rs pallets/gear-builtin/src/weights.rs
sed -i -E 's/\w+::WeightInfo for SubstrateWeight/WeightInfo for SubstrateWeight/' pallets/gear-builtin/src/weights.rs
# generate code for lightweight scheduler that is used in gtest and other crates
./scripts/weight-dump.sh
- name: "ACTIONS: Upload artifact with benchmarking errors (if exist)"
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: benchmarking-errors
path: scripts/benchmarking/benchmarking_errors.txt
if-no-files-found: ignore
- name: "ACTIONS: Upload artifact with updated weights (vara)"
uses: actions/upload-artifact@v4
with:
name: weights-vara
path: |
runtime/vara/src/weights/
utils/wasm-instrument/src/gas_metering/
- name: "TIP: download the artifacts and use `./scripts/unpack-weights.sh`"
run: |
echo "::notice::You can download the artifacts and use script \`./scripts/unpack-weights.sh\` to apply the changes to your branch"
- name: Create commit
if: ${{ inputs.change-type == 'commit' }}
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: >-
chore(runtime): update weights
file_pattern: "pallets/gear/src/ pallets/gear-builtin/src/ runtime/vara/src/weights/ utils/wasm-instrument/src/gas_metering/"
- name: Create Pull Request
if: ${{ inputs.change-type == 'pull_request' }}
uses: peter-evans/create-pull-request@v6
with:
add-paths: |
pallets/gear/src/
pallets/gear-builtin/src/
runtime/vara/src/weights/
utils/wasm-instrument/src/gas_metering/
commit-message: update weights
branch: weights/patch
branch-suffix: short-commit-hash
title: >-
chore(runtime): update weights
body: |
`run_all_benchmarks.sh` script is applied to update weights (branch: `${{ github.ref_name }}`, commit ${{ github.sha }})
**Note:** If CI fails, try manually updating the heuristic tests: `runtime/vara/src/tests.rs`
labels: |
A0-pleasereview
A4-insubstantial
destroy-runner:
if: always()
needs: [provision-runner, benchmarks]
runs-on: ubuntu-latest
steps:
- name: AWS Login
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Checkout
uses: actions/checkout@v4
with:
sparse-checkout: terraform
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
- name: Terraform Destroy
timeout-minutes: 60
run: |
cd terraform/bench-runner
terraform init
terraform destroy -auto-approve -var="aws_region=${{ needs.provision-runner.outputs.available_region }}" -var="instance_type=${{ inputs.instance_type }}" -var="instance_disk_size=${{ inputs.instance_disk_size }}" -var="registration_token=${{ env.REGISTER_TOKEN }}" -var="github_run_id=${{ github.run_id }}"
- name: Notify on Failure
if: failure()
uses: appleboy/telegram-action@master
with:
to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }}
token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }}
format: markdown
disable_web_page_preview: true
message: |
*Status*: 🔥
*Problem*: Issue with server destroying - bench-runner-${{ github.run_id }}
*Details*: Check [Benchmark Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})