From 01273f3e172a6a564bd02a021fe8403e03e16f92 Mon Sep 17 00:00:00 2001 From: Patrick Maslana Date: Wed, 25 Oct 2023 13:05:03 -0700 Subject: [PATCH] Testing Fuzzing --- .github/workflows/audit-check.yml | 14 - .github/workflows/build-arm64-wheels.yml | 121 -------- .github/workflows/build-crate.yml | 66 ---- .github/workflows/build-m1-wheel.yml | 182 ----------- .github/workflows/build-riscv.yml | 81 ----- .github/workflows/build-test.yml | 374 ++--------------------- .github/workflows/extensive-tests.yml | 120 -------- .github/workflows/npm-publish.yml | 88 ------ 8 files changed, 20 insertions(+), 1026 deletions(-) delete mode 100644 .github/workflows/audit-check.yml delete mode 100644 .github/workflows/build-arm64-wheels.yml delete mode 100644 .github/workflows/build-crate.yml delete mode 100644 .github/workflows/build-m1-wheel.yml delete mode 100644 .github/workflows/build-riscv.yml delete mode 100644 .github/workflows/extensive-tests.yml delete mode 100644 .github/workflows/npm-publish.yml diff --git a/.github/workflows/audit-check.yml b/.github/workflows/audit-check.yml deleted file mode 100644 index f27072dfd..000000000 --- a/.github/workflows/audit-check.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Security audit -on: - push: - paths: - - '**/Cargo.toml' - - '**/Cargo.lock' -jobs: - security_audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/audit-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-arm64-wheels.yml b/.github/workflows/build-arm64-wheels.yml deleted file mode 100644 index 6d8a5f72c..000000000 --- a/.github/workflows/build-arm64-wheels.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Build ARM64 wheels on ubuntu-latest - -on: - push: - branches: - - main - - dev - release: - types: [published] - pull_request: - branches: - - '**' - -permissions: - id-token: write - contents: read - -jobs: - build_wheels: - name: Build ARM64 Python Wheels - runs-on: [ARM64, Linux] - container: - image: ghcr.io/chia-network/build-images/centos-pypa-rust-aarch64:latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Set up - run: | - echo "${PATH}" - yum -y install openssl-devel - source /root/.cargo/env - rustup default stable - rustup target add aarch64-unknown-linux-musl - rm -rf venv - export PATH="${PATH}:/opt/python/cp39-cp39/bin/:/opt/python/cp38-cp38/bin/:/opt/python/cp37-cp37m/bin/" - - name: Build Python wheels - run: | - /opt/python/cp38-cp38/bin/python -m venv venv - if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi - . ./activate - pip install maturin==1.1.0 - CC=gcc maturin build --release --strip --manylinux 2014 - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: wheels - path: target/wheels/ - - - name: Clean up AMR64 - run: | - rm -rf venv - rm -rf dist - publish_wheels: - name: Publish ARM64 Python Wheels - runs-on: ubuntu-latest - needs: build_wheels - steps: - - name: Fetch wheels from previous job - uses: actions/download-artifact@v3 - with: - name: wheels - path: target/wheels/ - - - name: Install job deps - run: | - if [ ! -f "venv" ]; then rm -rf venv; fi - sudo apt install python3 python3-pip -y - python3 -m venv venv - if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi - . ./activate - pip3 install setuptools_rust - - name: Test for secrets access - id: check_secrets - shell: bash - run: | - unset HAS_AWS_SECRET - - if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi - echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT - env: - AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - - - name: Set Env - uses: Chia-Network/actions/setjobenv@main - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: publish (PyPi) - if: env.RELEASE == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels/ - skip-existing: true - - - name: publish (Test PyPi) - if: env.PRE_RELEASE == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - packages-dir: target/wheels/ - skip-existing: true - - - name: Configure AWS credentials - if: steps.check_secrets.outputs.HAS_AWS_SECRET - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload - aws-region: us-west-2 - - - name: Publish Dev - if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' - run: | - FILES=$(find ${{ github.workspace }}/target/wheels -type f -name '*.whl') - while IFS= read -r file; do - filename=$(basename $file) - aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename" - done <<< "$FILES" diff --git a/.github/workflows/build-crate.yml b/.github/workflows/build-crate.yml deleted file mode 100644 index cdd79aab4..000000000 --- a/.github/workflows/build-crate.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Taken from clvm_rs' version. -name: Build crate - -on: - push: - branches: - - main - - dev - tags: - - '**' - pull_request: - branches: - - '**' - -jobs: - build_crate: - name: Crate - runs-on: ubuntu-latest - strategy: - fail-fast: false - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up rusts - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt, clippy - - - name: Set up rust (stable) - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt, clippy - - - name: fmt (stable) - run: cargo +stable fmt -- --files-with-diff --check - - name: clippy (stable) - run: cargo +stable clippy - - name: tests - run: cargo test && cargo test --release - # The design of 'run' causes it to drop main.sym unannounced when compiling. - # ensure that it gets removed before cargo runs. - - name: remove stray main.sym - run: rm -f main.sym - - name: build - run: cargo build --release - - - name: dry-run of `cargo publish` - run: cargo publish --dry-run - - - name: Upload crate artifacts - uses: actions/upload-artifact@v3 - with: - name: crate - path: ./target/package/clvm_tools_rs-*.crate - - # this has not been tested, so probably needs to be debugged next time a tag is created - - name: publish to crates.io if tagged - if: startsWith(github.event.ref, 'refs/tags') - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.cargo_registry_token }} - run: cargo publish diff --git a/.github/workflows/build-m1-wheel.yml b/.github/workflows/build-m1-wheel.yml deleted file mode 100644 index a7a3c0a4e..000000000 --- a/.github/workflows/build-m1-wheel.yml +++ /dev/null @@ -1,182 +0,0 @@ -name: Build M1 Wheels - -on: - push: - branches: - - main - - dev - release: - types: [published] - pull_request: - branches: - - '**' - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}--${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - build_wheels: - name: Build wheel on Mac M1 - runs-on: [MacOS, ARM64] - strategy: - fail-fast: false - - steps: - - uses: Chia-Network/actions/clean-workspace@main - - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up rust - run: | - curl https://static.rust-lang.org/rustup/dist/aarch64-apple-darwin/rustup-init.sha256 | awk '{print $1 " *rustup-init"}' > checksum.txt - curl -O https://static.rust-lang.org/rustup/dist/aarch64-apple-darwin/rustup-init - cat checksum.txt - shasum -a 256 -c checksum.txt - - - name: Install rust - run: | - chmod +x rustup-init - ./rustup-init -y || (echo "Rust is already installed. Exiting..." && exit 2) - rm rustup-init checksum.txt - - - name: Build m1 wheels - run: | - python3 -m venv venv - . ./venv/bin/activate - export PATH=~/.cargo/bin:$PATH - pip install maturin==1.1.0 - maturin build -i python --release --strip - cargo test - - - name: Install clvm_tools_rs wheel - run: | - . ./venv/bin/activate - ls target/wheels/ - pip install ./target/wheels/clvm_tools_rs*.whl - - - name: Install other wheels - run: | - . ./venv/bin/activate - python -m pip install pytest - python -m pip install blspy - - - name: install clvm & clvm_tools - run: | - . ./venv/bin/activate - git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch - python -m pip install ./clvm - python -m pip install clvm_rs - - git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch - python -m pip install ./clvm_tools - - - name: Ensure clvm, clvm_rs, clvm_tools are installed - run: | - . ./venv/bin/activate - python -c 'import clvm' - python -c 'import clvm; print(clvm.__file__)' - python -c 'import clvm_rs; print(clvm_rs.__file__)' - python -c 'import clvm_tools; print(clvm_tools.__file__)' - python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' - - - name: Install pytest - run: | - . ./venv/bin/activate - pip install pytest pytest-asyncio - -# Cost tests are currently failing. -# - name: Run tests from clvm -# run: | -# . ./venv/bin/activate -# cd clvm -# python -m py.test tests - - - name: Run tests from clvm_tools - run: | - . ./venv/bin/activate - cd clvm_tools - pytest - - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: ./target/wheels - - upload: - name: Upload to PyPI - runs-on: ubuntu-latest - needs: build_wheels - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set Env - uses: Chia-Network/actions/setjobenv@main - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install python - uses: Chia-Network/actions/setup-python@main - with: - python-version: "3.10" - - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: wheels - path: ./target/wheels/ - - - name: publish (PyPi) - if: env.RELEASE == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels/ - skip-existing: true - - - name: publish (Test PyPi) - if: env.PRE_RELEASE == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - packages-dir: target/wheels/ - skip-existing: true - - - name: Test for secrets access - id: check_secrets - shell: bash - run: | - unset HAS_AWS_SECRET - - if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi - echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT - env: - AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - - - name: Configure AWS credentials - if: steps.check_secrets.outputs.HAS_AWS_SECRET - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload - aws-region: us-west-2 - - - name: Publish Dev - if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' - shell: bash - working-directory: ./target/wheels - run: | - FILES=$(find . -type f -name '*.whl') - while IFS= read -r file; do - filename=$(basename $file) - aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename" - done <<< "$FILES" diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml deleted file mode 100644 index 7fef28d9d..000000000 --- a/.github/workflows/build-riscv.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: QEMU Build and test riscv64 crate - -on: - push: - branches: - - main - - dev - tags: - - '**' - pull_request: - branches: - - '**' - -jobs: - build_crate: - name: Build riscv64 crate and run tests - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-latest ] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Set up QEMU on x86_64 - id: qemu - uses: docker/setup-qemu-action@v2 - with: - platforms: riscv64 - - - name: Build and Test - run: | - docker run --rm --platform linux/riscv64 \ - -v ${{ github.workspace }}:/ws --workdir=/ws \ - chianetwork/ubuntu-22.04-risc-builder:latest \ - bash -exc '\ - cargo test --release - ' - - build_wheels: - name: QEMU riscv64 wheel - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-latest ] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Set up QEMU on x86_64 - id: qemu - uses: docker/setup-qemu-action@v2 - with: - platforms: riscv64 - - - name: Build - run: | - docker run --rm --platform linux/riscv64 \ - -v ${{ github.workspace }}:/ws --workdir=/ws \ - chianetwork/ubuntu-22.04-risc-builder:latest \ - bash -exc '\ - pyenv global 3.10 - python -m venv venv && \ - source ./venv/bin/activate && \ - pip install --upgrade pip && \ - pip install --extra-index-url https://pypi.chia.net/simple/ maturin==1.2.3 && \ - maturin build -i python --release --strip \ - ' - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: packages - path: ./target/wheels/ - diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b3f9b9f7a..54f9b7cd4 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -4,8 +4,7 @@ name: Build on: push: branches: - - base - - dev + - '**' release: types: [published] pull_request: @@ -17,297 +16,14 @@ permissions: contents: read jobs: - build_wheels: - name: Wheel on ${{ matrix.os }} py-${{ matrix.python }} + fuzz_targets: + name: Run fuzz targets runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python: [3.8, 3.9, '3.10', 3.11] - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v4 - name: Install Python ${{ matrix.python }} - with: - python-version: ${{ matrix.python }} - - - name: Update pip - run: | - python -m pip install --upgrade pip - - - name: Set up rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Install dependencies - run: | - python -m pip install maturin==1.1.0 - - - name: Build MacOs with maturin on Python ${{ matrix.python }} - if: startsWith(matrix.os, 'macos') - env: - MACOSX_DEPLOYMENT_TARGET: '11.0' - run: | - python -m venv venv - ln -s venv/bin/activate - . ./activate - maturin build -i python --release --strip - - - name: Build Linux in manylinux2010 with maturin on Python ${{ matrix.python }} - if: startsWith(matrix.os, 'ubuntu') - run: | - docker run --rm \ - -v ${{ github.workspace }}:/ws --workdir=/ws \ - ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64 \ - bash -exc '\ - yum -y install libc6 openssl-devel && \ - source $HOME/.cargo/env && \ - rustup target add x86_64-unknown-linux-musl && \ - rm -rf venv && \ - PY_VERSION=${{ matrix.python }} - PY_VERSION=${PY_VERSION/.} && \ - echo "Python version with dot removed is $PY_VERSION" && \ - if [ "$PY_VERSION" = "37" ]; \ - then export SCND_VERSION="${PY_VERSION}m"; \ - else export SCND_VERSION="$PY_VERSION"; fi && \ - echo "Exporting path /opt/python/cp$PY_VERSION-cp$SCND_VERSION/bin" && \ - export PATH=/opt/python/cp$PY_VERSION-cp$SCND_VERSION/bin/:$PATH && \ - /opt/python/cp38-cp38/bin/python -m venv venv && \ - if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi && \ - . ./activate && \ - pip install --upgrade pip - ' - docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v1.1.0 build --release --strip --manylinux 2014 - # Refresh in case any ownerships changed. - mv target target.docker && cp -r target.docker target - # Ensure an empty .cargo-lock file exists. - touch target/release/.cargo-lock - - - name: Build Windows with maturin on Python ${{ matrix.python }} - if: startsWith(matrix.os, 'windows') - run: | - python -m venv venv - echo ". .\venv\Scripts\Activate.ps1" > activate.ps1 - . ./activate - maturin build -i python --release --strip - # Find and install the newly built wheel - python support/wheelname.py - - - name: Install clvm_tools_rs wheel - if: ${{ !startsWith(matrix.os, 'windows') }} - run: | - . ./activate - ls target/wheels/ - # this mess puts the name of the `.whl` file into `$WHEEL_PATH` - # remove the dot, use the `glob` lib to grab the file from the directory - export WHEEL_PATH=$(echo ${{ matrix.python }} | python -c 'DOTLESS=input().replace(".", ""); import glob; print(" ".join(filter(lambda x: "musl" not in x, glob.glob("target/wheels/clvm_tools_rs-*-cp*-*.whl"))))' ) - echo ${WHEEL_PATH} - pip install ${WHEEL_PATH} - - - name: Install other wheels - run: | - . ./activate - python -m pip install pytest - python -m pip install blspy - - - name: install clvm & clvm_tools - run: | - . ./activate - git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch - python -m pip install ./clvm - - echo "installing clvm_rs via pip" - pip install clvm_rs - - echo "installing clvm_tools for clvm tests" - - # Ensure clvm_tools is installed from its own repo. - git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch - python -m pip install ./clvm_tools - - - name: Ensure clvm, clvm_rs, clvm_tools_rs are installed - run: | - . ./activate - python -c 'import clvm' - python -c 'import clvm; print(clvm.__file__)' - python -c 'import clvm_rs; print(clvm_rs.__file__)' - python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' - echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV" - - - - name: Verify recompilation of old sources match with new compiler - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - set -x - . ./activate - # Build cmd line tools - PYO3_PYTHON=`which python` cargo build --no-default-features --release - - # Grab chia-blockchain - rm -rf chia-blockchain - git clone https://github.com/Chia-Network/chia-blockchain - - # Check that recompiling deployed puzzles match with their deployed hashes - cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain - (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip && \ - python -m pip install maturin==1.1.0 && \ - cd .. && python support/wheelname.py && \ - cd chia-blockchain && \ - # deps for manage_clvm.py - pip install click typing_extensions chia_rs clvm && \ - export PYTHONPATH=${PYTHONPATH}:$(pwd) && \ - ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - - - name: Test Classic command line tools with pytest - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - python -m pip install pytest - # This script installs the wheel built during this workflow. - python support/wheelname.py - # TODO: bring in the other tests in resources/tests/cmdline/tests besides cmds_test.py - (cd resources/tests/cmdline/tests && py.test cmds_test.py ) - - - name: Verify recompilation of cl21 sources - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - . ./activate - # We need chia-rs for the following. - pip install chia-rs - # Ensure we're using the sources we have. This is a hedge against - # changes made by any other step doing operations on pip. - # This script installs the wheel built during this workflow. - python support/wheelname.py - (cd resources/tests && python check-recompiles.py) - - - name: Verify recompilation follows date and modification rules - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - python support/wheelname.py - python resources/tests/test-clvm-recompile-behavior.py - - - name: Install pytest - run: | - . ./activate - pip install pytest pytest-asyncio - - - name: Run tests from clvm - run: | - . ./activate - cd clvm - pytest tests - - - name: Run tests from clvm_tools - run: | - . ./activate - cd clvm_tools - pytest tests - - - name: Run tests - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: cargo test --no-default-features - - - name: Exhaustive assign tests - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: cargo test -- --include-ignored assign - - - name: Check coverage - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - cargo install grcov - rustup component add llvm-tools-preview - python ./resources/coverage/run_coverage.py --require-percent 60 - - - name: Build alpine wheel via docker - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - cd resources/alpine && docker build -t clvm-tools-rs-alpine . - docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: wheels - path: ./target/wheels/ - - upload: - name: Upload to PyPI - runs-on: ubuntu-latest - needs: build_wheels - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set Env - uses: Chia-Network/actions/setjobenv@main - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install python - uses: Chia-Network/actions/setup-python@main - with: - python-version: "3.10" - - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: wheels - path: ./target/wheels/ - - - name: Publish distribution to PyPI - if: env.RELEASE == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels/ - skip-existing: true - - - name: Publish distribution to Test PyPI - if: env.PRE_RELEASE == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - packages-dir: target/wheels/ - skip-existing: true - - - name: Test for secrets access - id: check_secrets - shell: bash - run: | - unset HAS_AWS_SECRET - - if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi - echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT - env: - AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - - - name: Configure AWS credentials - if: steps.check_secrets.outputs.HAS_AWS_SECRET - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload - aws-region: us-west-2 - - - name: Publish Dev - if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' - shell: bash - working-directory: ./target/wheels - run: | - FILES=$(find . -type f -name '*.whl') - while IFS= read -r file; do - filename=$(basename $file) - aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename" - done <<< "$FILES" - - fmt: - runs-on: ubuntu-20.04 - name: cargo fmt + os: [macos-latest, ubuntu-latest] + python: [3.7] steps: - uses: actions/checkout@v3 with: @@ -315,73 +31,23 @@ jobs: - name: Install rust uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: nightly override: true components: rustfmt, clippy - name: fmt - run: cargo fmt -- --files-with-diff --check - - clippy: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: clippy - override: true + run: | + cd fuzz + cargo fmt -- --files-with-diff --check - name: clippy - run: cargo clippy --all -- -D warnings - - uses: giraffate/clippy-action@v1 - with: - reporter: 'github-pr-review' - github_token: ${{ secrets.GITHUB_TOKEN }} - - unit_tests: - runs-on: ubuntu-20.04 - name: Unit tests - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt, clippy - - name: cargo test - run: cargo test - - coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - name: Run for coverage run: | - sudo apt-get update - sudo apt-get install lcov -y - rustup component add llvm-tools-preview - cargo install grcov - export RUSTFLAGS="-Cinstrument-coverage" - export LLVM_PROFILE_FILE=$(pwd)/target/clvm_tools_rs-%p-%m.profraw - export CARGO_TARGET_DIR=$(pwd)/target - cargo test --release --workspace - python -m venv venv - source venv/bin/activate - git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch - pip install ./clvm_tools - pip install maturin pytest - maturin develop --release - (cd resources/tests/cmdline/tests && pytest) - grcov . --binary-path target -s . --branch --ignore-not-existing --ignore='*/.cargo/*' --ignore='*/tests/*' -o rust_cov.info - python -c 'with open("rust_cov.info") as f: lines = [l for l in f if not (l.startswith("DA:") and int(l.split(",")[1].strip()) >= 2**63)]; open("lcov.info", "w").writelines(lines)' - - name: Upload to Coveralls - uses: coverallsapp/github-action@v2 - if: always() - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - with: - path-to-lcov: './lcov.info' + cd fuzz + cargo clippy + - name: cargo-fuzz + run: cargo +nightly install cargo-fuzz + # TODO: it would be nice to save and restore the corpus between runs + - name: build corpus + run: | + cd tools + cargo run --bin generate-fuzz-corpus + - name: build + run: cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=30 || exit 255" diff --git a/.github/workflows/extensive-tests.yml b/.github/workflows/extensive-tests.yml deleted file mode 100644 index 6bf997da7..000000000 --- a/.github/workflows/extensive-tests.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Taken from clvm_rs' version. -name: Extensive tests - -on: - push: - branches: - - main - - dev - tags: - - '**' - pull_request: - branches: - - '**' - -jobs: - extensive_tests: - name: Extensive tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up rusts - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt, clippy - - - name: Set up rust (stable) - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt, clippy - - - uses: actions/setup-python@v4 - name: Install Python 3.11 - with: - python-version: 3.11 - - - name: Update pip - run: | - python -m pip install --upgrade pip - - - name: Set up rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Install dependencies - run: | - python -m pip install maturin==1.1.0 - - - name: Build Linux in manylinux2010 with maturin on Python ${{ matrix.python }} - run: | - podman run --rm=true \ - -v ${{ github.workspace }}:/ws:rw --workdir=/ws \ - ghcr.io/chia-network/build-images/centos-pypa-rust-x86_64 \ - bash -exc '\ - yum -y install libc6 openssl-devel && \ - source $HOME/.cargo/env && \ - rustup target add x86_64-unknown-linux-musl && \ - rm -rf venv && \ - PY_VERSION=${{ matrix.python }} - PY_VERSION=${PY_VERSION/.} && \ - echo "Python version with dot removed is $PY_VERSION" && \ - if [ "$PY_VERSION" = "37" ]; \ - then export SCND_VERSION="${PY_VERSION}m"; \ - else export SCND_VERSION="$PY_VERSION"; fi && \ - echo "Exporting path /opt/python/cp$PY_VERSION-cp$SCND_VERSION/bin" && \ - export PATH=/opt/python/cp$PY_VERSION-cp$SCND_VERSION/bin/:$PATH && \ - /opt/python/cp38-cp38/bin/python -m venv venv && \ - if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi && \ - . ./activate && \ - pip install --upgrade pip - ' - docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v1.1.0 build --release --strip --manylinux 2014 - # Refresh in case any ownerships changed. - mv target target.docker && cp -r target.docker target - # Ensure an empty .cargo-lock file exists. - touch target/release/.cargo-lock - - - name: Install clvm_tools_rs wheel - if: ${{ !startsWith(matrix.os, 'windows') }} - run: | - . ./activate - ls target/wheels/ - # this mess puts the name of the `.whl` file into `$WHEEL_PATH` - # remove the dot, use the `glob` lib to grab the file from the directory - export WHEEL_PATH=$(echo ${{ matrix.python }} | python -c 'DOTLESS=input().replace(".", ""); import glob; print(" ".join(filter(lambda x: "musl" not in x, glob.glob("target/wheels/clvm_tools_rs-*-cp*-*.whl"))))' ) - echo ${WHEEL_PATH} - pip install ${WHEEL_PATH} - - - name: Install other wheels - run: | - . ./activate - python -m pip install pytest - python -m pip install blspy - - - name: install clvm & clvm_tools - run: | - . ./activate - git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch - python -m pip install ./clvm - - echo "installing clvm_rs via pip" - pip install clvm_rs - - echo "installing clvm_tools for clvm tests" - # clvm tools is required to run the tests is clvm - python -m pip install clvm_tools - - - name: Run game referee test - run: | - . ./activate - cp support/test-game-referee.sh . - sh test-game-referee.sh resources/tests/game-referee-in-cl21 diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml deleted file mode 100644 index 46a2d9fe6..000000000 --- a/.github/workflows/npm-publish.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: npm publish - -on: - push: - branches: - - base - release: - types: [published] - pull_request: - branches: - - '**' - -concurrency: - # SHA is added to the end if on `main` to let all main workflows run - group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ (github.ref == 'refs/heads/main') && github.sha || '' }} - cancel-in-progress: true - -jobs: - build_npm: - name: Npm - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set Env - uses: Chia-Network/actions/setjobenv@main - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up rusts - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt, clippy - - - name: install wasm-pack - run: cargo install --version 0.11.1 wasm-pack - - - name: wasm-pack build and pack - run: wasm-pack build --release --target=nodejs wasm && wasm-pack pack wasm - - - name: Setup Node 16.x - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - # Cargo.toml won't allow an "@" in the name, so we just update the package name this way for NPM - - name: Update package name for npm - working-directory: ${{ github.workspace }}/wasm/pkg - run: | - cp package.json package.json.orig - jq '.name="@chia/chialisp"' package.json > temp.json && mv temp.json package.json - - - name: Test wasm - run: node wasm/tests/index.js - - - name: Test clvm-js like wasm interface - run: | - cd wasm/tests/clvm-tools-interface && npm install && yarn test - - - name: Upload npm pkg artifacts - uses: actions/upload-artifact@v3 - with: - name: npm-pkg - path: ./wasm/pkg/clvm_tools_wasm-*.tgz - - - name: Test for secrets access - id: check_secrets - shell: bash - run: | - unset HAS_SECRET - if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi - echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT - env: - SECRET: "${{ secrets.NPM_TOKEN }}" - - - name: Publish wasm - if: env.FULL_RELEASE == 'true' && steps.check_secrets.outputs.HAS_SECRET - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - working-directory: ${{ github.workspace }}/wasm/pkg - run: | - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc - rm -f clvm_tools_wasm-*.tgz - npm publish --access public