From 81014421f3135956893eb6ba878b57b534034c9c Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Wed, 7 Aug 2024 07:12:16 +1000 Subject: [PATCH] CI: Re-write using maintainer tools Re-write CI using the new maintainer tools script. A few things to note: - Put sanitizer and wasm jobs in their own scripts - Utilize `extra_tests.sh` for additional feature combos - We are exceeding the 20 job limit, see the README Unless I'm made a mistake this shouldn't reduce the test coverage in any way. --- .github/workflows/README.md | 43 +++ .../cron-semi-weekly-update-nightly.yml | 41 +++ .github/workflows/rust.yml | 312 ++++++++++++++---- Cargo.toml | 2 +- contrib/_test.sh | 126 ------- contrib/crates.sh | 8 + contrib/extra_tests.sh | 64 ++++ contrib/no-std.sh | 46 +++ contrib/sanitizer.sh | 70 ++++ contrib/test.sh | 40 --- contrib/test_vars.sh | 14 + contrib/wasm.sh | 14 + justfile | 28 +- nightly-version | 1 + 14 files changed, 566 insertions(+), 243 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/cron-semi-weekly-update-nightly.yml delete mode 100755 contrib/_test.sh create mode 100755 contrib/crates.sh create mode 100755 contrib/extra_tests.sh create mode 100755 contrib/no-std.sh create mode 100755 contrib/sanitizer.sh delete mode 100755 contrib/test.sh create mode 100644 contrib/test_vars.sh create mode 100755 contrib/wasm.sh create mode 100644 nightly-version diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..c62b8fd4c --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,43 @@ +# rust-bitcoin workflow notes + +We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). + +ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration + +The minimal/recent lock files are handled by CI (`rust.yml`). + +## Jobs + +Run from `rust.yml` unless stated otherwise. Total 21 jobs but +`Prepare` is quick and must be run first anyway. + +0. `Prepare` +1. `Stable - minimal` +2. `Stable - recent` +3. `Nightly - minimal` +4. `Nightly - recent` +5. `MSRV - minimal` +6. `MSRV - recent` +7. `Lint` +8. `Docs` +9. `Docsrs` +10. `Bench` +11. `Format` +12. `ASAN` +13. `Arch32Bit` +14. `WASM` +15. `NoStd` + +16. `Cross testing - aarch64-unknown-linux-gnu` +17. `Cross testing - i686-unknown-linux-gnu` +18. `Cross testing - x86_64-pc-windows-gnu` +19. `Cross testing - x86_64-unknown-linux-gnu` +20. `Cross testing - aarch64-unknown-linux-musl` +21. `Cross testing - arm-unknown-linux-gnueabi` +22. `Cross testing - arm-unknown-linux-gnueabihf` +23. `Cross testing - armv7-unknown-linux-gnueabihf` +24. `Cross testing - powerpc-unknown-linux-gnu` +25. `Cross testing - powerpc64le-unknown-linux-gnu` +26. `Cross testing - riscv64gc-unknown-linux-gnu` +27. `Cross testing - s390x-unknown-linux-gnu` +28. `Cross testing - x86_64-unknown-linux-musl` diff --git a/.github/workflows/cron-semi-weekly-update-nightly.yml b/.github/workflows/cron-semi-weekly-update-nightly.yml new file mode 100644 index 000000000..51f7afa2e --- /dev/null +++ b/.github/workflows/cron-semi-weekly-update-nightly.yml @@ -0,0 +1,41 @@ +name: Update Nightly rustc +on: + schedule: + - cron: "0 0 * * 1,4" # runs every Monday and Thursday at 00:00 + workflow_dispatch: # allows manual triggering +jobs: + format: + name: Update nightly rustc + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + - name: Update rust.yml to use latest nightly + run: | + set -x + # Not every night has a nightly, so extract the date from whatever + # version of the compiler dtolnay/rust-toolchain gives us. + NIGHTLY_DATE=$(rustc +nightly --verbose --version | sed -ne 's/^commit-date: //p') + # Update the nightly version in the reference file. + echo "nightly-${NIGHTLY_DATE}" > nightly-version + echo "nightly_date=${NIGHTLY_DATE}" >> $GITHUB_ENV + # Some days there is no new nightly. In this case don't make an empty PR. + if ! git diff --exit-code > /dev/null; then + echo "Updated nightly. Opening PR." + echo "changes_made=true" >> $GITHUB_ENV + else + echo "Attempted to update nightly but the latest-nightly date did not change. Not opening any PR." + echo "changes_made=false" >> $GITHUB_ENV + fi + - name: Create Pull Request + if: env.changes_made == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.APOELSTRA_CREATE_PR_TOKEN }} + author: Update Nightly Rustc Bot + committer: Update Nightly Rustc Bot + title: Automated daily update to rustc (to nightly-${{ env.nightly_date }}) + body: | + Automated update to Github CI workflow `rust.yml` by [create-pull-request](https://github.com/peter-evans/create-pull-request) GitHub action + commit-message: Automated update to Github CI to rustc nightly-${{ env.nightly_date }} + branch: create-pull-request/daily-nightly-update diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f95c8bbd0..b2b6a8c18 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,104 +1,292 @@ -on: [push, pull_request] +--- # rust-bitcoin CI: If you edit this file please update README.md +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - 'test-ci/**' + pull_request: name: Continuous integration jobs: - Stable: + Prepare: + runs-on: ubuntu-latest + outputs: + nightly_version: ${{ steps.read_toolchain.outputs.nightly_version }} + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Read nightly version" + id: read_toolchain + run: echo "nightly_version=$(cat nightly-version)" >> $GITHUB_OUTPUT + + Stable: # 2 jobs, one per lock file. name: Test - stable toolchain runs-on: ubuntu-latest strategy: fail-fast: false + matrix: + dep: [minimal, recent] steps: - - name: Checkout Crate - uses: actions/checkout@v3 - - name: Checkout Toolchain - # https://github.com/dtolnay/rust-toolchain + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Running test script - env: - DO_LINT: true - DO_DOCS: true - DO_FEATURE_MATRIX: true - run: ./contrib/test.sh + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh stable - Beta: - name: Test - beta toolchain + Nightly: # 2 jobs, one per lock file. + name: Test - nightly toolchain + needs: Prepare runs-on: ubuntu-latest strategy: fail-fast: false + matrix: + dep: [minimal, recent] steps: - - name: Checkout Crate - uses: actions/checkout@v3 - - name: Checkout Toolchain - uses: dtolnay/rust-toolchain@beta - - name: Running test script - run: ./contrib/test.sh + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh nightly - Nightly: - name: Test - nightly toolchain + MSRV: # 2 jobs, one per lock file. + name: Test - 1.63.0 toolchain runs-on: ubuntu-latest strategy: fail-fast: false + matrix: + dep: [minimal, recent] steps: - - name: Checkout Crate - uses: actions/checkout@v3 - - name: Checkout Toolchain + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.63.0" + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh msrv + + Lint: + name: Lint - nightly toolchain + needs: Prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: Install clippy + run: rustup component add clippy + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh lint + + Docs: + name: Docs - stable toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@stable + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh docs + + Docsrs: + name: Docs - nightly toolchain + needs: Prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh docsrs + + Bench: + name: Bench - nightly toolchain + needs: Prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + rev: b2ac115 + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh bench + + Format: + name: Format - nightly toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" uses: dtolnay/rust-toolchain@nightly - - name: Install src - run: rustup component add rust-src - - name: Install xargo - run: cargo install xargo - - name: Running test script - env: - DO_FMT: true - DO_BENCH: true - DO_DOCSRS: true - DO_ASAN: true - run: ./contrib/test.sh + - name: "Install rustfmt" + run: rustup component add rustfmt + - name: "Check formatting" + run: cargo +nightly fmt --all -- --check - MSRV: - name: Test - MSRV toolchain + ASAN: # 1 job, run sanitizer from script. + name: ASAN - nightly toolchain + needs: Prepare runs-on: ubuntu-latest strategy: fail-fast: false + matrix: + dep: [recent] steps: - - name: Checkout Crate - uses: actions/checkout@v3 - - name: Checkout Toolchain - uses: dtolnay/rust-toolchain@1.63.0 - - name: Running test script - env: - DO_FEATURE_MATRIX: true - run: ./contrib/test.sh + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Install rust-src" + run: rustup component add rust-src + - name: "Install xargo" + run: cargo install xargo + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run sanitizer script" + run: ./contrib/sanitizer.sh Arch32bit: name: Test 32-bit version runs-on: ubuntu-latest steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v3 - - name: Checkout Toolchain + - name: "Checkout toolchain" uses: dtolnay/rust-toolchain@stable - - name: Add architecture i386 + - name: "Add architecture i386" run: sudo dpkg --add-architecture i386 - - name: Install i686 gcc + - name: "Install i686 gcc" run: sudo apt-get update -y && sudo apt-get install -y gcc-multilib - - name: Install target + - name: "Install target" run: rustup target add i686-unknown-linux-gnu - - name: Run test on i686 + - name: "Run test on i686" run: cargo test --target i686-unknown-linux-gnu WASM: - name: WASM + name: WASM - stable toolchain runs-on: ubuntu-latest + strategy: + fail-fast: false + # Note we do not use the recent lock file for wasm testing. steps: - - name: Checkout Crate - uses: actions/checkout@v3 - - name: Install clang - run: sudo apt-get install -y clang - - name: Checkout Toolchain + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Running WASM tests - env: - DO_WASM: true - run: ./contrib/test.sh + - name: "Run wasm script" + run: ./contrib/wasm.sh + + NoStd: # 1 job, run no-std test from script. + name: no-std - nightly toolchain + needs: Prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Install src" + run: rustup component add rust-src + - name: "Install xargo" + run: cargo install xargo + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run no-std-test script" + run: ./contrib/no-std.sh diff --git a/Cargo.toml b/Cargo.toml index 59268c125..3ca983b22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,10 +35,10 @@ global-context-less-secure = ["global-context"] [dependencies] secp256k1-sys = { version = "0.10.0", default-features = false, path = "./secp256k1-sys" } -serde = { version = "1.0.103", default-features = false, optional = true } hashes = { package = "bitcoin_hashes", version = "0.14", default-features = false, optional = true } rand = { version = "0.8", default-features = false, optional = true } +serde = { version = "1.0.103", default-features = false, optional = true } [dev-dependencies] rand_core = "0.6" diff --git a/contrib/_test.sh b/contrib/_test.sh deleted file mode 100755 index d44b9507e..000000000 --- a/contrib/_test.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -REPO_DIR=$(git rev-parse --show-toplevel) -FEATURES="hashes global-context lowmemory rand recovery serde std alloc" - -cargo --version -rustc --version - -# Work out if we are using a nightly toolchain. -NIGHTLY=false -if cargo --version | grep nightly; then - NIGHTLY=true -fi - -# Pin dependencies as required if we are using MSRV toolchain. -if cargo --version | grep "1\.48"; then - cargo update -p wasm-bindgen-test --precise 0.3.34 - cargo update -p serde_test --precise 1.0.175 -fi - -# Test if panic in C code aborts the process (either with a real panic or with SIGILL) -cargo test -- --ignored --exact 'tests::test_panic_raw_ctx_should_terminate_abnormally' 2>&1 \ - | tee /dev/stderr \ - | grep "SIGILL\\|\[libsecp256k1] illegal argument. " - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Defaults / sanity checks -cargo build --locked --all -cargo test --locked --all - -if [ "$DO_FEATURE_MATRIX" = true ]; then - cargo build --locked --all --no-default-features - cargo test --locked --all --no-default-features - - # All features - cargo build --locked --all --no-default-features --features="$FEATURES" - cargo test --locked --all --no-default-features --features="$FEATURES" - # Single features - for feature in ${FEATURES} - do - cargo build --locked --all --no-default-features --features="$feature" - cargo test --locked --all --no-default-features --features="$feature" - done - # Features tested with 'std' feature enabled. - for feature in ${FEATURES} - do - cargo build --locked --all --no-default-features --features="std,$feature" - cargo test --locked --all --no-default-features --features="std,$feature" - done - # Other combos - RUSTFLAGS='--cfg=secp256k1_fuzz' RUSTDOCFLAGS='--cfg=secp256k1_fuzz' cargo test --locked --all - RUSTFLAGS='--cfg=secp256k1_fuzz' RUSTDOCFLAGS='--cfg=secp256k1_fuzz' cargo test --locked --all --features="$FEATURES" - cargo test --locked --all --features="rand serde" - - if [ "$NIGHTLY" = true ]; then - cargo test --locked --all --all-features - RUSTFLAGS='--cfg=secp256k1_fuzz' RUSTDOCFLAGS='--cfg=secp256k1_fuzz' cargo test --locked --all --all-features - fi - - # Examples - cargo run --locked --example sign_verify --features=hashes,std - cargo run --locked --example sign_verify_recovery --features=recovery,hashes,std - cargo run --locked --example generate_keys --features=rand,std -fi - -if [ "$DO_LINT" = true ] -then - cargo clippy --locked --all-features --all-targets -- -D warnings - cargo clippy --locked --example sign_verify --features=hashes,std -- -D warnings - cargo clippy --locked --example sign_verify_recovery --features=recovery,hashes,std -- -D warnings - cargo clippy --locked --example generate_keys --features=rand,std -- -D warnings -fi - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCSRS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features -fi - -# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command -# above this checks that we feature guarded docs imports correctly. -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features -fi - -# Address Sanitizer -if [ "$DO_ASAN" = true ]; then - clang --version - cargo clean - CC='clang -fsanitize=address -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ - cargo test --lib --all --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - cargo clean - # The -Cllvm-args=-msan-eager-checks=0 flag was added to overcome this issue: - # https://github.com/rust-bitcoin/rust-secp256k1/pull/573#issuecomment-1399465995 - CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes -Cllvm-args=-msan-eager-checks=0' \ - cargo test --lib --all --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - - cd no_std_test - xargo run --release --target=x86_64-unknown-linux-gnu | grep -q "Verified Successfully" - xargo run --release --target=x86_64-unknown-linux-gnu --features=alloc | grep -q "Verified alloc Successfully" - cd - -fi - -# Run formatter if told to. -if [ "$DO_FMT" = true ]; then - if [ "$NIGHTLY" = false ]; then - echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)" - exit 1 - fi - rustup component add rustfmt - cargo fmt --check || exit 1 -fi - -# Bench if told to, only works with non-stable toolchain (nightly, beta). -if [ "$DO_BENCH" = true ] -then - RUSTFLAGS='--cfg=bench' cargo bench --features=recovery,rand,std -fi - -exit 0 diff --git a/contrib/crates.sh b/contrib/crates.sh new file mode 100755 index 000000000..43e296b17 --- /dev/null +++ b/contrib/crates.sh @@ -0,0 +1,8 @@ +# No shebang, this file should not be executed. +# shellcheck disable=SC2148 +# +# disable verify unused vars, despite the fact that they are used when sourced +# shellcheck disable=SC2034 + +# Dot for single crate in workspace to test. +CRATES=(".") diff --git a/contrib/extra_tests.sh b/contrib/extra_tests.sh new file mode 100755 index 000000000..e05a519e5 --- /dev/null +++ b/contrib/extra_tests.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -ex + +REPO_DIR=$(git rev-parse --show-toplevel) + +# Make all cargo invocations verbose. +export CARGO_TERM_VERBOSE=true + +# Set to false to turn off verbose output. +flag_verbose=true + +# Use the current `Cargo.lock` file without updating it. +cargo="cargo --locked" + +main() { + source_test_vars # Get feature list. + local features="$FEATURES_WITH_STD" + + RUSTFLAGS='--cfg=secp256k1_fuzz' RUSTDOCFLAGS='--cfg=secp256k1_fuzz' $cargo test --locked --all + RUSTFLAGS='--cfg=secp256k1_fuzz' RUSTDOCFLAGS='--cfg=secp256k1_fuzz' $cargo test --locked --all --features="$features" + + $cargo test --all --features="rand serde" +} + +# ShellCheck can't follow non-constant source, `test_vars_script` is correct. +# shellcheck disable=SC1090 +source_test_vars() { + local test_vars_script="$REPO_DIR/contrib/test_vars.sh" + + verbose_say "Sourcing $test_vars_script" + + if [ -e "$test_vars_script" ]; then + # Set crate specific variables. + . "$test_vars_script" + else + err "Missing $test_vars_script" + fi +} + +say() { + echo "extra_tests: $1" +} + +say_err() { + say "$1" >&2 +} + +verbose_say() { + if [ "$flag_verbose" = true ]; then + say "$1" + fi +} + +err() { + echo "$1" >&2 + exit 1 +} + +# +# Main script +# +main "$@" +exit 0 diff --git a/contrib/no-std.sh b/contrib/no-std.sh new file mode 100755 index 000000000..c896bd893 --- /dev/null +++ b/contrib/no-std.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Run no-std tests, requires nightly toolchain. + +set -euox pipefail + +main() { + need_nightly + check_required_commands + + pushd no_std_test > /dev/null + + xargo run --release --target=x86_64-unknown-linux-gnu | grep -q "Verified Successfully" + xargo run --release --target=x86_64-unknown-linux-gnu --features=alloc | grep -q "Verified alloc Successfully" + + popd +} + +# Check all the commands we use are present in the current environment. +check_required_commands() { + need_cmd xargo +} + +need_cmd() { + if ! command -v "$1" > /dev/null 2>&1 + then err "need '$1' (command not found)" + fi +} + +err() { + echo "$1" >&2 + exit 1 +} + +need_nightly() { + cargo_ver=$(cargo --version) + if echo "$cargo_ver" | grep -q -v nightly; then + err "Need a nightly compiler; have $(cargo --version)" + fi +} + +# +# Main script +# +main "$@" +exit 0 diff --git a/contrib/sanitizer.sh b/contrib/sanitizer.sh new file mode 100755 index 000000000..d18cf7493 --- /dev/null +++ b/contrib/sanitizer.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# Run the Address/Memory Sanitizer tests. + +set -euox pipefail + +REPO_DIR=$(git rev-parse --show-toplevel) + +# Set to true to enable verbose output. +flag_verbose=false + +main() { + source_test_vars # Get feature list. + local features="$FEATURES_WITH_STD" + + clang --version + cargo clean + + CC='clang -fsanitize=address -fno-omit-frame-pointer' \ + RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ + ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ + cargo test --lib --all --features="$features" -Zbuild-std --target x86_64-unknown-linux-gnu + cargo clean + + # The -Cllvm-args=-msan-eager-checks=0 flag was added to overcome this issue: + # https://github.com/rust-bitcoin/rust-secp256k1/pull/573#issuecomment-1399465995 + CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ + RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes -Cllvm-args=-msan-eager-checks=0' \ + cargo test --lib --all --features="$features" -Zbuild-std --target x86_64-unknown-linux-gnu +} + +# ShellCheck can't follow non-constant source, `test_vars_script` is correct. +# shellcheck disable=SC1090 +source_test_vars() { + local test_vars_script="$REPO_DIR/contrib/test_vars.sh" + + verbose_say "Sourcing $test_vars_script" + + if [ -e "$test_vars_script" ]; then + # Set crate specific variables. + . "$test_vars_script" + else + err "Missing $test_vars_script" + fi +} + +say() { + echo "extra_tests: $1" +} + +say_err() { + say "$1" >&2 +} + +verbose_say() { + if [ "$flag_verbose" = true ]; then + say "$1" + fi +} + +err() { + echo "$1" >&2 + exit 1 +} + +# +# Main script +# +main "$@" +exit 0 diff --git a/contrib/test.sh b/contrib/test.sh deleted file mode 100755 index fa6c14a7e..000000000 --- a/contrib/test.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -REPO_DIR=$(git rev-parse --show-toplevel) -DEPS="recent minimal" - -# Webassembly stuff -# -# The wasm-pack command does not correctly pass args to cargo so we cannot use --locked and test -# with per-commited lockfiles (recent/minimal). Just run the WASM tests from here instead. -if [ "$DO_WASM" = true ]; then - clang --version - CARGO_TARGET_DIR=wasm cargo install --force wasm-pack - printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml - CC=clang wasm-pack build - CC=clang wasm-pack test --node - - exit 0 -fi - -for dep in $DEPS -do - cp "Cargo-$dep.lock" Cargo.lock - $REPO_DIR/contrib/_test.sh - - if [ "$dep" = recent ]; - then - # We always test committed dependencies but we want to warn if they could've been updated - cargo update - if diff Cargo-recent.lock Cargo.lock; - then - echo "Dependencies are up to date" - else - echo "::warning file=Cargo-recent.lock::Dependencies could be updated" - fi - fi -done - -exit 0 diff --git a/contrib/test_vars.sh b/contrib/test_vars.sh new file mode 100644 index 000000000..0bbc175a5 --- /dev/null +++ b/contrib/test_vars.sh @@ -0,0 +1,14 @@ +# No shebang, this file should not be executed. +# shellcheck disable=SC2148 +# +# disable verify unused vars, despite the fact that they are used when sourced +# shellcheck disable=SC2034 + +# Test all these features with "std" enabled. +FEATURES_WITH_STD="hashes global-context global-context-less-secure lowmemory rand recovery serde" + +# Test all these features without "std" enabled. +FEATURES_WITHOUT_STD="hashes global-context global-context-less-secure lowmemory rand recovery serde alloc" + +# Run these examples. +EXAMPLES="sign_verify:hashes,std sign_verify_recovery:hashes,std,recovery generate_keys:rand,std" diff --git a/contrib/wasm.sh b/contrib/wasm.sh new file mode 100755 index 000000000..3d8bd8b9b --- /dev/null +++ b/contrib/wasm.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Run the WASM tests. +# +# The wasm-pack command does not correctly pass args to cargo so we cannot use --locked and test +# with per-commited lockfiles (recent/minimal). Just run the WASM tests from here instead. + +set -euox pipefail + +clang --version +CARGO_TARGET_DIR=wasm cargo install --force wasm-pack +printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml +CC=clang wasm-pack build +CC=clang wasm-pack test --node diff --git a/justfile b/justfile index d98737dc2..79f4edf87 100644 --- a/justfile +++ b/justfile @@ -3,36 +3,36 @@ default: # Cargo build everything. build: - cargo build --all-targets --all-features + cargo build --workspace --all-targets --all-features # Cargo check everything. check: - cargo check --all-targets --all-features + cargo check --workspace --all-targets --all-features # Lint everything. lint: - cargo clippy --all-targets --all-features -- --deny warnings + cargo +$(cat ./nightly-version) clippy --workspace --all-targets --all-features -- --deny warnings + +# Run cargo fmt +fmt: + cargo +$(cat ./nightly-version) fmt --all # Check the formatting format: - cargo +nightly fmt --check + cargo +$(cat ./nightly-version) fmt --all --check # Quick and dirty CI useful for pre-push checks. sane: lint - cargo test --quiet --all-targets --no-default-features > /dev/null || exit 1 - cargo test --quiet --all-targets > /dev/null || exit 1 - cargo test --quiet --all-targets --all-features > /dev/null || exit 1 + cargo test --quiet --workspace --all-targets --no-default-features > /dev/null || exit 1 + cargo test --quiet --workspace --all-targets > /dev/null || exit 1 + cargo test --quiet --workspace --all-targets --all-features > /dev/null || exit 1 # doctests don't get run from workspace root with `cargo test`. - cargo test --quiet --doc || exit 1 + cargo test --quiet --workspace --doc || exit 1 # Make an attempt to catch feature gate problems in doctests - cargo test --manifest-path Cargo.toml --doc --no-default-features > /dev/null || exit 1 - -# Check for API changes. -check-api: - ./contrib/check-for-api-changes.sh + cargo test --doc --no-default-features > /dev/null || exit 1 -# Update the lock files. +# Update the recent and minimal lock files. update-lock-files: ./contrib/update-lock-files.sh diff --git a/nightly-version b/nightly-version new file mode 100644 index 000000000..c33b8d13f --- /dev/null +++ b/nightly-version @@ -0,0 +1 @@ +nightly-2024-08-04