diff --git a/.github/workflows/crypto3-testing-linux.yml b/.github/workflows/crypto3-testing-linux.yml new file mode 100644 index 000000000..acd1f0100 --- /dev/null +++ b/.github/workflows/crypto3-testing-linux.yml @@ -0,0 +1,210 @@ +name: Crypto3 Build and Test on Linux Platforms + +on: + workflow_call: + inputs: + concurrency: + type: number + description: "Concurrency level (0 to use number of virtual cores)" + required: false + default: 0 + targets: + type: string + description: "Make and CTest targets. If not specified, everything is tested" + required: false + test-paths: + type: string + description: "Folders from which the test must be run" + required: true + boost-version: + type: string + description: "Version of Boost to install" + required: false + default: "1.81.0" +env: + TESTS_ARTIFACT_NAME: "test-results" + EVENT_FILE_ARTIFACT_NAME: "event-file" + +jobs: + upload-event-file: + # Needed to link test results with PR workflow run + name: "Upload Event File" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: ${{ env.EVENT_FILE_ARTIFACT_NAME }} + path: ${{ github.event_path }} + + build-and-test: + name: "Build and Test using Matrix" + runs-on: [self-hosted, Linux, X64, aws_autoscaling] + + env: + CONTAINER_TMP: /opt/ + HOST_TMP: /home/runner/work/_temp/ + DEBIAN_FRONTEND: noninteractive + + container: + image: ubuntu:22.04 + volumes: + - /home/runner/work/_temp/:/opt/ + + strategy: + # Set up a matrix to run the following 2 configurations: + # 1. + # 2. + matrix: + cpp_compiler: [g++, clang++-12] + build_type: [Release] + + steps: + - name: Install dependencies + run: | + env && \ + apt update && \ + apt install -y \ + build-essential \ + libssl-dev \ + cmake \ + clang-12 \ + git \ + libicu-dev \ + curl \ + pkg-config \ + libc-ares-dev \ + liblz4-dev \ + libgnutls28-dev \ + libprotobuf-dev \ + libyaml-cpp-dev \ + lksctp-tools \ + libsctp-dev \ + ragel \ + spd \ + lsb-release + + - name: Print toolchain information + run: | + git --version + cc --version + cmake --version + + - name: Get OS version + # Turn input strings (such as the build output directory) into step outputs. These step outputs can be used throughout the workflow file. + id: os_version + shell: bash + run: | + # Get platform version + platform_version=$(lsb_release -rs) + echo "platform-version=${platform_version}" >> $GITHUB_OUTPUT + + - name: Install boost + uses: MarkusJx/install-boost@v2.4.1 + id: install-boost + with: + # A list of supported versions can be found here: + # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json + boost_version: ${{ inputs.boost-version }} + platform_version: ${{ steps.os_version.outputs.platform-version }} + boost_install_dir: ${{ env.CONTAINER_TMP }} + toolset: gcc + arch: x86 + + - name: Checkout Crypto3 repository + # We need full history, because during CMake config stage we are finding the nearest tag + uses: actions/checkout@v4 + with: + repository: NilFoundation/crypto3 + fetch-depth: 1 # Fetch only the latest commit on the triggered branch/ref + submodules: false + + # Workaround: https://github.com/actions/checkout/issues/1169 + - name: Mark directory as safe + run: | + git config --system --add safe.directory $PWD + + - name: Checkout submodules + run: | + git submodule update --init --recursive --depth=1 + + - name: Set usefull strings + # Turn input strings (such as the build output directory) into step outputs. These step outputs can be used throughout the workflow file. + id: strings + shell: bash + run: | + docker_workspace_path="$PWD" + echo "docker-workspace-path=$docker_workspace_path" >> "$GITHUB_OUTPUT" + + if [ "${{ inputs.concurrency }}" = "0" ]; then + echo "Setting concurrency to number of logical cores" + proc_number=$(nproc) + else + echo "Setting concurrency to user-defined value" + proc_number=${{ inputs.concurrency }} + fi + + echo "proc-number=${proc_number}" >> $GITHUB_OUTPUT + + - name: Clean index.lock files if checkout step was cancelled or failed + if: cancelled() || failure() + run: | + find .git -name 'index.lock' -exec rm -v {} \; + + - name: Configure CMake + run: > + cmake -B build + -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }} + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DBUILD_TESTS=TRUE + -DENABLE_JUNIT_TEST_OUTPUT=TRUE -S ${{ steps.strings.outputs.docker-workspace-path }} + + env: + BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" + + - name: Build tests + id: build + working-directory: ${{ steps.strings.outputs.docker-workspace-path }}/build + run: | + # Not considering failed targets bad. We will handle them as junit test result + build_log_path=${{ steps.strings.outputs.docker-workspace-path }}/build/build.log + targets_str=$(echo "${{ inputs.targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') + make -k -j ${{ steps.strings.outputs.proc-number }} ${targets_str} 2>&1 | tee ${build_log_path} + echo "build-log=$build_log_path" >> "$GITHUB_OUTPUT" + + - name: Generate JUnit Report from build result + id: make-build-report + uses: NilFoundation/ci-cd/actions/build-log-to-junit@v1 + with: + build-log: ${{ steps.build.outputs.build-log }} + + - name: Run tests + id: run_tests + working-directory: ${{ steps.strings.outputs.docker-workspace-path }}/build + # Not considering failed tests bad. We will compare diff instead + run: | + artifact_dir=${{ steps.strings.outputs.docker-workspace-path }}/../results_for_uploading_${{github.sha}} + mkdir -p $artifact_dir + artifact_dir=$(cd ${{ steps.strings.outputs.docker-workspace-path }}/../results_for_uploading_${{github.sha}} && pwd) + echo "artifact-dir=$artifact_dir" >> "$GITHUB_OUTPUT" + + custom_tests_dir=$artifact_dir/ubuntu-22.04/${{ matrix.cpp_compiler }}/${{ matrix.build_type }} + mkdir -p $custom_tests_dir + + targets_str=$(echo "${{ inputs.targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' '|' | sed 's/|$//') + ctest -v -j ${{ steps.strings.outputs.proc-number }} -R "(${targets_str})" || true + + test_paths="${{ inputs.test-paths }}" + for dir in $(echo "${test_paths}" | awk 'NF {$1=$1; print}') + do + mkdir -p $custom_tests_dir/$dir + mv ${{ steps.strings.outputs.docker-workspace-path }}/build/$dir/junit_results/* $custom_tests_dir/$dir + done + + mv ${{ steps.make-build-report.outputs.build-junit-report }} $custom_tests_dir + + - name: Upload tests JUnit results + uses: actions/upload-artifact@v3 + with: + name: ${{ env.TESTS_ARTIFACT_NAME }} + path: ${{ steps.run_tests.outputs.artifact-dir }} diff --git a/.github/workflows/crypto3-testing-mac.yml b/.github/workflows/crypto3-testing-mac.yml new file mode 100644 index 000000000..77bed62fc --- /dev/null +++ b/.github/workflows/crypto3-testing-mac.yml @@ -0,0 +1,174 @@ +name: Crypto3 Build and Test on Mac + +on: + workflow_call: + inputs: + concurrency: + type: number + description: "Concurrency level (0 to use number of virtual cores)" + required: false + default: 0 + targets: + type: string + description: "Make and CTest targets. If not specified, everything is tested" + required: false + test-paths: + type: string + description: "Folders from which the test must be run" + required: true + boost-version: + type: string + description: "Version of Boost to install" + required: false + default: '1.81.0' # The least version supported by both matrix.os +env: + TESTS_ARTIFACT_NAME: 'test-results' + EVENT_FILE_ARTIFACT_NAME: 'event-file' + +jobs: + upload-event-file: + # Needed to link test results with PR workflow run + name: "Upload Event File" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: ${{ env.EVENT_FILE_ARTIFACT_NAME }} + path: ${{ github.event_path }} + + build-and-test: + name: "Build and Test using Matrix" + runs-on: [macos-12] + strategy: + # Set up a matrix to run the following 2 configurations: + # 1. + # 2. + matrix: + cpp_compiler: [g++, clang++] + build_type: [Release] + steps: + - name: Install homebrew + run: > + /bin/bash -c "$(curl -fsSL + https://raw.githubusercontent.com/Homebrew/install/master/install.sh + )" + + - name: Run brew install + id: brew-install + run: | + brew install \ + cmake \ + icu4c \ + bash + + - name: Print toolchain information + run: | + git --version + cc --version + cmake --version + bash --version + which -a bash + echo PATH: $PATH + + - name: Checkout Crypto3 repository + uses: actions/checkout@v4 + with: + repository: NilFoundation/crypto3 + submodules: 'true' # Using true fetches only the top-level submodules + fetch-depth: 1 # Fetch only the latest commit on the triggered branch/ref + + - name: Set usefull strings + # Turn input strings (such as the build output directory) into step outputs. These step outputs can be used throughout the workflow file. + id: strings + shell: bash + run: | + # Dependencies dir location + echo "dependencies-dir=${{ github.workspace }}/../dependencies" >> "$GITHUB_OUTPUT" + + # Get platform version + platform_version=$(sw_vers -productVersion | awk -F '.' '{print $1}') + echo "platform-version=${platform_version}" >> $GITHUB_OUTPUT + + if [ "${{ inputs.concurrency }}" = "0" ]; then + echo "Setting concurrency to number of logical cores" + proc_number=$(sysctl -n hw.logicalcpu) + else + echo "Setting concurrency to user-defined value" + proc_number=${{ inputs.concurrency }} + fi + + echo "proc-number=${proc_number}" >> $GITHUB_OUTPUT + + - name: Set up dependencies directory + run: | + mkdir -p "${{ steps.strings.outputs.dependencies-dir }}" + + - name: Install boost + uses: MarkusJx/install-boost@v2.4.4 + id: install-boost + with: + boost_version: ${{ inputs.boost-version }} + # A list of supported versions can be found here: + # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json + platform_version: ${{ steps.strings.outputs.platform-version }} + boost_install_dir: ${{ steps.strings.outputs.dependencies-dir }} + + - name: Configure CMake + run: > + cmake -B build + -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }} + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DBUILD_TESTS=TRUE + -DENABLE_JUNIT_TEST_OUTPUT=TRUE + -S ${{ github.workspace }} + + env: + BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" + + - name: Build tests + id: build + working-directory: ${{ github.workspace }}/build + run: | + # Not considering failed targets bad. We will handle them as junit test result + build_log_path=${{ github.workspace }}/build/build.log + targets_str=$(echo "${{ inputs.targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') + make -k -j ${{ steps.strings.outputs.proc-number }} ${targets_str} 2>&1 | tee ${build_log_path} + echo "build-log=$build_log_path" >> "$GITHUB_OUTPUT" + + - name: Generate JUnit Report from build result + id: make-build-report + uses: NilFoundation/ci-cd/actions/build-log-to-junit@v1 + with: + build-log: ${{ github.workspace }}/build/build.log + + - name: Run tests + id: run_tests + working-directory: ${{ github.workspace }}/build + # Not considering failed tests bad. We will compare diff instead + run: | + artifact_dir=${{ github.workspace }}/../results_for_uploading_${{github.sha}} + mkdir -p $artifact_dir + artifact_dir=$(cd ${{ github.workspace }}/../results_for_uploading_${{github.sha}} && pwd) + echo "artifact-dir=$artifact_dir" >> "$GITHUB_OUTPUT" + + custom_tests_dir=$artifact_dir/macos-12/${{ matrix.cpp_compiler }}/${{ matrix.build_type }} + mkdir -p $custom_tests_dir + + targets_str=$(echo "${{ inputs.targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' '|' | sed 's/|$//') + ctest -v -j ${{ steps.strings.outputs.proc-number }} -R "(${targets_str})" || true + + test_paths="${{ inputs.test-paths }}" + for dir in $(echo "${test_paths}" | awk 'NF {$1=$1; print}') + do + mkdir -p $custom_tests_dir/$dir + mv ${{ github.workspace }}/build/$dir/junit_results/* $custom_tests_dir/$dir + done + + mv ${{ steps.make-build-report.outputs.build-junit-report }} $custom_tests_dir + + - name: Upload tests JUnit results + uses: actions/upload-artifact@v3 + with: + name: ${{ env.TESTS_ARTIFACT_NAME }} + path: ${{ steps.run_tests.outputs.artifact-dir }} diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml new file mode 100644 index 000000000..fe59679b1 --- /dev/null +++ b/.github/workflows/publish-results.yml @@ -0,0 +1,13 @@ +# Needed to publish test results in fork +name: Testing Callback + +on: + workflow_run: + workflows: ["PR Testing"] + types: + - completed + +jobs: + call-reusable-workflow: + name: Call Reusable Testing Callback Workflow + uses: NilFoundation/ci-cd/.github/workflows/reusable-crypto3-publish-result.yml@v1.2.0 diff --git a/.github/workflows/pull-request-action.yml b/.github/workflows/pull-request-action.yml new file mode 100644 index 000000000..d48465ac1 --- /dev/null +++ b/.github/workflows/pull-request-action.yml @@ -0,0 +1,33 @@ +name: Testing for mac and linux + +on: + workflow_call: + inputs: + targets: + type: string + description: "Make and CTest targets. If not specified, everything is tested" + required: false + test-paths: + type: string + description: "Folders from which the test must be ran" + required: true +jobs: + matrix-test-linux: + name: Linux Crypto3 Testing + uses: ./.github/workflows/crypto3-testing-linux.yml + + secrets: inherit + with: + targets: ${{ inputs.targets }} + test-paths: ${{ inputs.test-paths }} + + matrix-test-mac: + name: Mac Crypto3 Testing + uses: ./.github/workflows/crypto3-testing-mac.yml + + secrets: inherit + with: + # TODO(martun): fix this sometime soon. All the targets must work on mac. + targets: crypto3_zk_math_expression_test # ${{ inputs.targets }} + test-paths: libs/zk/test # ${{ inputs.test-paths }} + diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 000000000..a6676f75d --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,70 @@ +name: PR Testing + +on: + pull_request: + types: + - opened + - synchronize + +jobs: + run-pull-request-actions: + name: Crypto3 Testing + uses: ./.github/workflows/pull-request-action.yml + + secrets: inherit + with: + targets: | + crypto3_zk_commitment_fold_polynomial_test + crypto3_zk_commitment_fri_test + crypto3_zk_commitment_lpc_test + crypto3_zk_systems_plonk_placeholder_placeholder_test + crypto3_zk_commitment_powers_of_tau_test + crypto3_zk_commitment_proof_of_knowledge_test + crypto3_zk_commitment_r1cs_gg_ppzksnark_mpc_test + crypto3_zk_math_expression_test + crypto3_zk_systems_plonk_plonk_constraint_test + marshalling_fri_commitment_test + marshalling_lpc_commitment_test + marshalling_placeholder_common_data_test + marshalling_placeholder_proof_test + marshalling_sparse_vector_test + marshalling_plonk_constraint_system_test + marshalling_r1cs_gg_ppzksnark_primary_input_test + marshalling_r1cs_gg_ppzksnark_proof_test + marshalling_r1cs_gg_ppzksnark_verification_key_test + algebra_curves_test + algebra_fields_test + algebra_hash_to_curve_test + algebra_pairing_test + math_expression_test + math_lagrange_interpolation_test + math_polynomial_test + math_polynomial_dfs_test + math_polynomial_arithmetic_test + math_polynomial_view_test + hash_blake2b_test + hash_crc_test + hash_find_group_hash_test + hash_h2c_test + hash_h2f_test + hash_keccak_test + hash_md4_test + hash_md5_test + hash_pack_test + hash_pedersen_test + hash_ripemd_test + hash_sha1_test + hash_sha2_test + hash_sha3_test + hash_sha_test + hash_static_digest_test + hash_tiger_test + hash_poseidon_test + + test-paths: | + libs/zk/test + libs/algebra/test + libs/hash/test + libs/math/test + libs/marshalling/zk/test + diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml deleted file mode 100644 index bf8e0b9d1..000000000 --- a/.github/workflows/run_tests.yml +++ /dev/null @@ -1,190 +0,0 @@ -name: Run tests - -on: - # Let's run it on any commit - [pull_request] - # Triggers the workflow on pull request events but only for the master branch - #pull_request: - # branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - #workflow_dispatch: - -env: - SUITE_REPO: "NilFoundation/crypto3" - CACHE_NAME: "checkout-job-cache" - -jobs: - checkout: - runs-on: [self-hosted, tests-runner] - steps: - - name: Cleanup # TODO - move to scripts on runner - run: | - rm -rf ./* || true - rm -rf ./.??* || true - - - name: Checkout suite - uses: actions/checkout@v3 - with: - repository: ${{ env.SUITE_REPO }} - - submodules: recursive - - - name: Cmake and build - env: - CMAKE_ARGS: " - -DCMAKE_BUILD_TYPE=Debug - -DBUILD_SHARED_LIBS=FALSE - -DBUILD_TESTS=TRUE - -DZK_PLACEHOLDER_PROFILING=TRUE - " - run: | - mkdir build - cd build - cmake ${{ env.CMAKE_ARGS }} .. - - - name: Archive build results - run: | - touch ${{ env.CACHE_NAME }}.tar.gz - tar -czf ${{ env.CACHE_NAME }}.tar.gz --exclude=${{ env.CACHE_NAME }}.tar.gz . - - - name: Cache archived job output - uses: actions/upload-artifact@v3 - with: - name: ${{ env.CACHE_NAME }} - path: ${{ env.CACHE_NAME }}.tar.gz - retention-days: 1 - - run_tests: - runs-on: [self-hosted, tests-runner] - needs: [checkout] - strategy: - fail-fast: false - matrix: - include: - - target: crypto3_zk_commitment_fold_polynomial_test - path: libs/zk/test - - target: crypto3_zk_commitment_fri_test - path: libs/zk/test - - target: crypto3_zk_commitment_lpc_test - path: libs/zk/test - - target: crypto3_zk_systems_plonk_placeholder_placeholder_test - path: libs/zk/test - - target: crypto3_zk_commitment_powers_of_tau_test - path: libs/zk/test - - target: crypto3_zk_commitment_proof_of_knowledge_test - path: libs/zk/test - - target: crypto3_zk_commitment_r1cs_gg_ppzksnark_mpc_test - path: libs/zk/test - - target: crypto3_zk_math_expression_test - path: libs/zk/test - - target: crypto3_zk_systems_plonk_plonk_constraint_test - path: libs/zk/test - - - target: marshalling_fri_commitment_test - path: libs/marshalling/zk/test - - target: marshalling_lpc_commitment_test - path: libs/marshalling/zk/test - - target: marshalling_placeholder_common_data_test - path: libs/marshalling/zk/test - - target: marshalling_placeholder_proof_test - path: libs/marshalling/zk/test - - target: marshalling_sparse_vector_test - path: libs/marshalling/zk/test - - target: marshalling_plonk_constraint_system_test - path: libs/marshalling/zk/test - - target: marshalling_r1cs_gg_ppzksnark_primary_input_test - path: libs/marshalling/zk/test - - target: marshalling_r1cs_gg_ppzksnark_proof_test - path: libs/marshalling/zk/test - - target: marshalling_r1cs_gg_ppzksnark_verification_key_test - path: libs/marshalling/zk/test - - - target: algebra_curves_test - path: libs/algebra/test - - target: algebra_fields_test - path: libs/algebra/test - - target: algebra_hash_to_curve_test - path: libs/algebra/test - - target: algebra_pairing_test - path: libs/algebra/test - - - target: math_expression_test - path: libs/math/test - - target: math_lagrange_interpolation_test - path: libs/math/test - - target: math_polynomial_test - path: libs/math/test - - target: math_polynomial_dfs_test - path: libs/math/test - - target: math_polynomial_arithmetic_test - path: libs/math/test - - target: math_polynomial_view_test - path: libs/math/test - - - target: hash_blake2b_test - path: libs/hash/test - - target: hash_crc_test - path: libs/hash/test - - target: hash_find_group_hash_test - path: libs/hash/test - - target: hash_h2c_test - path: libs/hash/test - - target: hash_h2f_test - path: libs/hash/test - - target: hash_keccak_test - path: libs/hash/test - - target: hash_md4_test - path: libs/hash/test - - target: hash_md5_test - path: libs/hash/test - - target: hash_pack_test - path: libs/hash/test - - target: hash_pedersen_test - path: libs/hash/test - - target: hash_ripemd_test - path: libs/hash/test - - target: hash_sha1_test - path: libs/hash/test - - target: hash_sha2_test - path: libs/hash/test - - target: hash_sha3_test - path: libs/hash/test - - target: hash_sha_test - path: libs/hash/test - - target: hash_static_digest_test - path: libs/hash/test - - target: hash_tiger_test - path: libs/hash/test - - # https://github.com/NilFoundation/crypto3-hash/issues/100 - # - target: hash_reinforced_concrete_test - # path: libs/hash/test - - steps: - - name: Cleanup # TODO - move to scripts on runner - run: | - rm -rf ./* || true - rm -rf ./.??* || true - - - name: Upload checkout job cache - uses: actions/download-artifact@v3 - with: - name: ${{ env.CACHE_NAME }} - - - name: Extract artifacts - run: | - tar -xf ${{ env.CACHE_NAME }}.tar.gz - rm ${{ env.CACHE_NAME }}.tar.gz - - - name: Build - working-directory: ./build - run: cmake --build . -t ${{ matrix.target }} - - - name: Run test - working-directory: ./build - run: | - cd ${{ matrix.path }} - COLOR='\033[0;33m' - echo -e "${COLOR}${{ matrix.target }}" - ./${{ matrix.target }} diff --git a/libs/marshalling/core b/libs/marshalling/core index e3e785773..6b1940baa 160000 --- a/libs/marshalling/core +++ b/libs/marshalling/core @@ -1 +1 @@ -Subproject commit e3e785773334258022a67b9e28fc2e65ce6805dc +Subproject commit 6b1940baad808b447796950c414e003061418b94 diff --git a/libs/marshalling/zk b/libs/marshalling/zk index c99144f1e..3a8efcf62 160000 --- a/libs/marshalling/zk +++ b/libs/marshalling/zk @@ -1 +1 @@ -Subproject commit c99144f1ece940289cf7155836ea01381a15dff8 +Subproject commit 3a8efcf62eba21e8b842518717ecfb9f0c2b1e81 diff --git a/libs/multiprecision b/libs/multiprecision index 8a1327c2b..da5ee0ae3 160000 --- a/libs/multiprecision +++ b/libs/multiprecision @@ -1 +1 @@ -Subproject commit 8a1327c2b4cbed38c3da2c0e2caed16bd3bb30d7 +Subproject commit da5ee0ae399ada85cf568e09b9bd7da1cf034643