From bc17a2b33c4c88f186e9c6d666595bd994263c87 Mon Sep 17 00:00:00 2001 From: Laszlo Voros Date: Mon, 5 Aug 2024 11:43:59 +0200 Subject: [PATCH] Improve CI Test with jit-no-reg-alloc Signed-off-by: Laszlo Voros --- .github/workflows/actions.yml | 697 ++++++++++++++++-------------- test/wasmBenchmarker/benchmark.py | 28 +- tools/run-tests.py | 51 ++- 3 files changed, 421 insertions(+), 355 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index cdde9d39b..a1d23b4de 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -2,9 +2,9 @@ name: Actions on: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] env: RUNNER: tools/run-tests.py @@ -13,258 +13,270 @@ jobs: check-tidy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo add-apt-repository "deb http://mirrors.kernel.org/ubuntu/ focal main universe" - sudo apt update - sudo apt install -y clang-format-10 - - name: Test - run: tools/check_tidy.py + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo add-apt-repository "deb http://mirrors.kernel.org/ubuntu/ focal main universe" + sudo apt update + sudo apt install -y clang-format-10 + - name: Test + run: tools/check_tidy.py build-on-macos: runs-on: macos-13 steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - brew update - brew install cmake ninja pkg-config - - name: Build x64 - env: - BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - cmake -H. -Bout/mac $BUILD_OPTIONS - ninja -Cout/mac - - name: Run Tests - run: | - #FIXME try-catch is unstable in macos build - #remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by apple-clang - rm $GITHUB_WORKSPACE/test/wasm-spec/core/call.wast - rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast - $RUNNER --engine="$GITHUB_WORKSPACE/out/mac/walrus" + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + brew update + brew install cmake ninja pkg-config + - name: Build x64 + env: + BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja + run: | + cmake -H. -Bout/mac $BUILD_OPTIONS + ninja -Cout/mac + - name: Run Tests + run: | + #FIXME try-catch is unstable in macos build + #remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by apple-clang + rm $GITHUB_WORKSPACE/test/wasm-spec/core/call.wast + rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast + $RUNNER --engine="$GITHUB_WORKSPACE/out/mac/walrus" build-on-macos-arm64: runs-on: macos-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - brew update - brew install cmake ninja - - name: Build arm64 - env: - BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - cmake -H. -Bout/mac $BUILD_OPTIONS - ninja -Cout/mac - - name: Run Tests - run: | - #FIXME try-catch is unstable in macos build - #remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by apple-clang - rm $GITHUB_WORKSPACE/test/wasm-spec/core/call.wast - rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast - $RUNNER --engine="$GITHUB_WORKSPACE/out/mac/walrus" + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + brew update + brew install cmake ninja + - name: Build arm64 + env: + BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja + run: | + cmake -H. -Bout/mac $BUILD_OPTIONS + ninja -Cout/mac + - name: Run Tests + run: | + #FIXME try-catch is unstable in macos build + #remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by apple-clang + rm $GITHUB_WORKSPACE/test/wasm-spec/core/call.wast + rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast + $RUNNER --engine="$GITHUB_WORKSPACE/out/mac/walrus" build-by-clang: + strategy: + matrix: + arch: + - x86 + - x64 + compile: + - name: Build + options: "" + dir: clang + tests: "" + - name: Pure build + options: -DWALRUS_WASI=OFF + dir: "pure" + tests: "basic-tests wasm-test-core jit" + switch: + - --jit + - --jit-no-reg-alloc + - "" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - - name: Build x86 - env: - BUILD_OPTIONS: -DWALRUS_ARCH=x86 -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - CC=clang CXX=clang++ cmake -H. -Bout/clang/x86 $BUILD_OPTIONS - ninja -Cout/clang/x86 - - name: Build x64 - env: - BUILD_OPTIONS: -DWALRUS_ARCH=x64 -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - CC=clang CXX=clang++ cmake -H. -Bout/clang/x64 $BUILD_OPTIONS - ninja -Cout/clang/x64 - - name: Pure Build x86 - env: - BUILD_OPTIONS: -DWALRUS_ARCH=x86 -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja - run: | - CC=clang CXX=clang++ cmake -H. -Bout/pure/x86 $BUILD_OPTIONS - ninja -Cout/pure/x86 - - name: Pure Build x64 - env: - BUILD_OPTIONS: -DWALRUS_ARCH=x64 -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja - run: | - CC=clang CXX=clang++ cmake -H. -Bout/pure/x64 $BUILD_OPTIONS - ninja -Cout/pure/x64 - - name: Run Tests - run: | - # remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by clang - rm $GITHUB_WORKSPACE/test/wasm-spec/core/call.wast - rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast - $RUNNER --engine="$GITHUB_WORKSPACE/out/clang/x86/walrus" - $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/clang/x64/walrus" - $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/pure/x86/walrus" basic-tests wasm-test-core jit - $RUNNER --engine="$GITHUB_WORKSPACE/out/pure/x64/walrus" basic-tests wasm-test-core jit + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + - name: ${{ matrix.compile.name }} ${{ matrix.arch }} + env: + BUILD_OPTIONS: -DWALRUS_ARCH=${{ matrix.arch }} -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell ${{ matrix.compile.options }} -GNinja + run: | + CC=clang CXX=clang++ cmake -H. -Bout/${{ matrix.compile.dir }}/${{ matrix.arch }} $BUILD_OPTIONS + ninja -Cout/${{ matrix.compile.dir }}/${{ matrix.arch }} + - name: Run Tests + run: | + # remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by clang + rm $GITHUB_WORKSPACE/test/wasm-spec/core/call.wast + rm $GITHUB_WORKSPACE/test/wasm-spec/core/call_indirect.wast + $RUNNER ${{ matrix.switch }} --engine="$GITHUB_WORKSPACE/out/${{ matrix.compile.dir }}/${{ matrix.arch }}/walrus" ${{ matrix.compile.tests }} - build-test-on-x86: + build-test-on-x86_x64: + strategy: + matrix: + arch: + - x86 + - x64 + switch: + - --jit + - --jit-no-reg-alloc + - "" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - - name: Build x86 - env: - BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - cmake -H. -Bout/linux/x86 $BUILD_OPTIONS - ninja -Cout/linux/x86 - - name: Run Tests - run: | - $RUNNER --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus" - $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/linux/x86/walrus" + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + - name: Build ${{ matrix.arch }} + env: + BUILD_OPTIONS: -DWALRUS_ARCH=${{ matrix.arch }} -DWALRUS_HOST=linux -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja + run: | + cmake -H. -Bout/linux/${{ matrix.arch }} $BUILD_OPTIONS + ninja -Cout/linux/${{ matrix.arch }} + - name: Run Tests + run: | + $RUNNER ${{ matrix.switch }} --engine="$GITHUB_WORKSPACE/out/linux/${{ matrix.arch }}/walrus" - build-test-on-x64: + build-on-x64-with-perf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - - name: Build x64 - env: - BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - cmake -H. -Bout/linux/x64 $BUILD_OPTIONS - ninja -Cout/linux/x64 - - name: Run Tests - run: | - $RUNNER --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus" - $RUNNER --jit --engine="$GITHUB_WORKSPACE/out/linux/x64/walrus" - - build-on-x64-with-perf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - - name: Build x64 - env: - BUILD_OPTIONS: -DWALRUS_JITPERF=1 -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - run: | - cmake -H. -Bout/linux/x64 $BUILD_OPTIONS - ninja -Cout/linux/x64 + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + - name: Build x64 + env: + BUILD_OPTIONS: -DWALRUS_JITPERF=1 -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja + run: | + cmake -H. -Bout/linux/x64 $BUILD_OPTIONS + ninja -Cout/linux/x64 build-test-on-armv7: + strategy: + matrix: + switch: + - --jit + - --jit-no-reg-alloc + - "" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Build in arm32 container - uses: uraimo/run-on-arch-action@v2.8.1 - with: - arch: armv7 - distro: ubuntu_latest - - # Install deps into the container. With the token, the container will be cached - # The image is cached publically like a package - githubToken: ${{ github.token }} - - install: | - apt-get update - apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git - run: | - CC=clang CXX=clang++ cmake -H. -Bout/debug -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja - CC=clang CXX=clang++ cmake -H. -Bout/pure -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja - ninja -Cout/debug - ninja -Cout/pure + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install compiler + run: | + sudo apt update + sudo apt install -y make wget ninja-build libglib2.0-dev + wget https://toolchains.bootlin.com/downloads/releases/toolchains/armv7-eabihf/tarballs/armv7-eabihf--glibc--stable-2024.05-1.tar.xz + tar -xvf armv7-eabihf--glibc--stable-2024.05-1.tar.xz + - name: Install qemu + run: | + wget https://download.qemu.org/qemu-9.0.2.tar.xz + tar -xvf qemu-9.0.2.tar.xz + cd qemu-9.0.2 + ./configure --target-list="arm-linux-user" + make -j4 + - name: Build in armv7 + env: + CC: ./armv7-eabihf--glibc--stable-2024.05-1/bin/arm-buildroot-linux-gnueabihf-gcc-13.3.0.br_real + CXX: ./armv7-eabihf--glibc--stable-2024.05-1/bin/arm-buildroot-linux-gnueabihf-g++.br_real + run: | + cmake -H. -DCMAKE_EXE_LINKER_FLAGS="-static" -Bout/arm/debug -DCMAKE_BUILD_TYPE=debug -DWALRUS_ARCH=arm -DWALRUS_HOST=linux -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja + cmake -H. -DCMAKE_EXE_LINKER_FLAGS="-static" -Bout/arm/pure -DWALRUS_ARCH=arm -DWALRUS_HOST=linux -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja + ninja -C out/arm/debug/ + ninja -C out/arm/pure/ + - name: Test in armv7 + run: | # remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by clang rm ./test/wasm-spec/core/call.wast rm ./test/wasm-spec/core/call_indirect.wast - python3 ./tools/run-tests.py --engine="./out/debug/walrus" - python3 ./tools/run-tests.py --jit --engine="./out/debug/walrus" - python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit - python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit + python3 ./tools/run-tests.py ${{ matrix.switch }} --engine="./out/debug/walrus" --qemu="./qemu-9.0.2/build/qemu-arm" + python3 ./tools/run-tests.py ${{ matrix.switch }} --engine="./out/pure/walrus" --qemu="./qemu-9.0.2/build/qemu-arm" basic-tests wasm-test-core jit - build-test-on-arm64: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Build in arm64 container - uses: uraimo/run-on-arch-action@v2.8.1 - with: - arch: aarch64 - distro: ubuntu22.04 + # build-test-on-armv7_aarch64: + # strategy: + # matrix: + # arch: + # - run_on_arch: armv7 + # walrus: arm + # distro: ubuntu_latest + # - run_on_arch: aarch64 + # walrus: aarch64 + # distro: ubuntu22.04 + # switch: + # - --jit + # - --jit-no-reg-alloc + # - "" + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Build in container + # uses: uraimo/run-on-arch-action@v2.7.2 + # with: + # arch: ${{ matrix.arch.run_on_arch }} + # distro: ${{ matrix.arch.distro }} - # Install deps into the container. With the token, the container will be cached - # The image is cached publically like a package - githubToken: ${{ github.token }} + # # Install deps into the container. With the token, the container will be cached + # # The image is cached publically like a package + # githubToken: ${{ github.token }} - install: | - apt-get update - apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git - run: | - CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja - CC=clang CXX=clang++ cmake -H. -Bout/pure -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja - ninja -Cout/release - ninja -Cout/pure - python3 ./tools/run-tests.py --engine="./out/release/walrus" - python3 ./tools/run-tests.py --jit --engine="./out/release/walrus" - python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit - python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit + # install: | + # apt-get update + # apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git + # run: | + # CC=clang CXX=clang++ cmake -H. -Bout/debug -DWALRUS_ARCH=${{ matrix.arch.walrus }} -DWALRUS_HOST=linux -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -GNinja + # CC=clang CXX=clang++ cmake -H. -Bout/pure -DWALRUS_ARCH=${{ matrix.arch.walrus }} -DWALRUS_HOST=linux -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja + # ninja -Cout/debug + # ninja -Cout/pure + # # remove 2 test files due to stack overflow occurred by recursion calls in debug mode build by clang + # rm ./test/wasm-spec/core/call.wast + # rm ./test/wasm-spec/core/call_indirect.wast + # python3 ./tools/run-tests.py ${{ matrix.switch }} --engine="./out/debug/walrus" + # python3 ./tools/run-tests.py ${{ matrix.switch }} --engine="./out/pure/walrus" basic-tests wasm-test-core jit build-test-on-riscv64: + strategy: + matrix: + switch: + # TODO enable JIT for RISC-V + #- --jit + #- --jit-no-reg-alloc + - "" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Build in riscv64 container - uses: uraimo/run-on-arch-action@v2.8.1 - with: - arch: riscv64 - distro: ubuntu22.04 + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build in riscv64 container + uses: uraimo/run-on-arch-action@v2.8.1 + with: + arch: riscv64 + distro: ubuntu22.04 - # Install deps into the container. With the token, the container will be cached - # The image is cached publically like a package - githubToken: ${{ github.token }} + # Install deps into the container. With the token, the container will be cached + # The image is cached publically like a package + githubToken: ${{ github.token }} - install: | - apt-get update - apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git - run: | - # TODO enable JIT for RISC-V - CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_JIT=OFF -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja - CC=clang CXX=clang++ cmake -H. -Bout/pure -DWALRUS_JIT=OFF -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja - ninja -Cout/release - ninja -Cout/pure - python3 ./tools/run-tests.py --engine="./out/release/walrus" - #python3 ./tools/run-tests.py --jit --engine="./out/release/walrus" - python3 ./tools/run-tests.py --engine="./out/pure/walrus" basic-tests wasm-test-core jit - #python3 ./tools/run-tests.py --jit --engine="./out/pure/walrus" basic-tests wasm-test-core jit + install: | + apt-get update + apt-get install -y cmake build-essential ninja-build pkg-config python3 clang git + run: | + CC=clang CXX=clang++ cmake -H. -Bout/release -DWALRUS_JIT=OFF -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja + CC=clang CXX=clang++ cmake -H. -Bout/pure -DWALRUS_JIT=OFF -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -DWALRUS_WASI=OFF -GNinja + ninja -Cout/release + ninja -Cout/pure + python3 ./tools/run-tests.py ${{ matrix.switch }} --engine="./out/release/walrus" + python3 ./tools/run-tests.py ${{ matrix.switch }} --engine="./out/pure/walrus" basic-tests wasm-test-core jit test-on-windows-x86-x64: runs-on: windows-2022 @@ -272,125 +284,150 @@ jobs: matrix: arch: [x86, x64] steps: - - name: Set git cllf config - run: | - git config --global core.autocrlf input - git config --global core.eol lf - - uses: actions/checkout@v4 - with: - submodules: true - - uses: lukka/get-cmake@latest - - uses: GuillaumeFalourd/setup-windows10-sdk-action@v2 - with: - sdk-version: 20348 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install msvc redist package - run: | - (new-object System.Net.WebClient).DownloadFile('https://github.com/abbodi1406/vcredist/releases/download/v0.73.0/VisualCppRedist_AIO_x86_x64.exe','VisualCppRedist_AIO_x86_x64.exe') - .\VisualCppRedist_AIO_x86_x64.exe /y - - uses: ilammy/msvc-dev-cmd@v1.13.0 - with: - arch: ${{ matrix.arch }} - sdk: "10.0.20348.0" - - name: Build ${{ matrix.arch }} Release - run: | - CMake -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION:STRING="10.0" -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.arch }} -Bout/ -G Ninja -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl -DCMAKE_BUILD_TYPE=release - CMake --build out/ --config Release - - name: Run tests - run: | - python tools\run-tests.py --engine=%cd%\out\walrus.exe - shell: cmd - - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 15 + - name: Set git cllf config + run: | + git config --global core.autocrlf input + git config --global core.eol lf + - uses: actions/checkout@v4 + with: + submodules: true + - uses: lukka/get-cmake@latest + - uses: GuillaumeFalourd/setup-windows10-sdk-action@v2 + with: + sdk-version: 20348 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install msvc redist package + run: | + (new-object System.Net.WebClient).DownloadFile('https://github.com/abbodi1406/vcredist/releases/download/v0.73.0/VisualCppRedist_AIO_x86_x64.exe','VisualCppRedist_AIO_x86_x64.exe') + .\VisualCppRedist_AIO_x86_x64.exe /y + - uses: ilammy/msvc-dev-cmd@v1.13.0 + with: + arch: ${{ matrix.arch }} + sdk: "10.0.20348.0" + - name: Build ${{ matrix.arch }} Release + run: | + CMake -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION:STRING="10.0" -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.arch }} -Bout/ -G Ninja -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl -DCMAKE_BUILD_TYPE=release + CMake --build out/ --config Release + - name: Run tests + run: | + python tools\run-tests.py --engine=%cd%\out\walrus.exe + shell: cmd + - if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 15 + + build-test-extended-feature: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + - name: Build x64 + env: + BUILD_OPTIONS: -DWALRUS_ARCH=x64 -DWALRUS_HOST=linux -DWALRUS_MODE=debug -DWALRUS_OUTPUT=shell -DWALRUS_EXTENDED_FEATURES=ON -GNinja + run: | + cmake -H. -Bout/extended $BUILD_OPTIONS + ninja -Cout/extended + - name: Run Tests + run: | + $RUNNER --engine="$GITHUB_WORKSPACE/out/extended/walrus" wasm-test-extended - build-test-performance: + build-test-performance-x64_x86: + strategy: + matrix: + arch: + - x64 + - x86 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - sudo pip install pandas - sudo pip install py-markdown-table - sudo pip install tqdm - - name: Build x64 - env: - BUILD_OPTIONS: -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja - run: | - cmake -H. -Bout/linux/x64 $BUILD_OPTIONS - ninja -Cout/linux/x64 - - name: Run Tests - run: | - test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/linux/x64/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + sudo pip install pandas + sudo pip install py-markdown-table + sudo pip install tqdm + - name: Build ${{ matrix.arch }} + env: + BUILD_OPTIONS: -DWALRUS_ARCH=${{ matrix.arch }} -DWALRUS_HOST=linux -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja + run: | + cmake -H. -Bout/linux/${{ matrix.arch }} $BUILD_OPTIONS + ninja -Cout/linux/${{ matrix.arch }} + - name: Run Tests + run: | + test/wasmBenchmarker/benchmark.py --engines $GITHUB_WORKSPACE/out/linux/${{ matrix.arch }}/walrus --iterations 2 --verbose --summary --results i j2i n2i j n2j built-test-wasm-c-api: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - - name: Build x64 - env: - BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=api_test -GNinja - run: | - cmake -H. -Bout/api_test/x64 $BUILD_OPTIONS - ninja -Cout/api_test/x64 - cp third_party/wasm-c-api/example/*.wasm out/api_test/x64/. - - name: Run Tests - working-directory: ./out/api_test/x64 - run: | - ./wasm-c-api-callback - ./wasm-c-api-global - ./wasm-c-api-hello - ./wasm-c-api-memory - ./wasm-c-api-multi - ./wasm-c-api-table + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + - name: Build x64 + env: + BUILD_OPTIONS: -DWALRUS_MODE=debug -DWALRUS_OUTPUT=api_test -GNinja + run: | + cmake -H. -Bout/api_test/x64 $BUILD_OPTIONS + ninja -Cout/api_test/x64 + cp third_party/wasm-c-api/example/*.wasm out/api_test/x64/. + - name: Run Tests + working-directory: ./out/api_test/x64 + run: | + ./wasm-c-api-callback + ./wasm-c-api-global + ./wasm-c-api-hello + ./wasm-c-api-memory + ./wasm-c-api-multi + ./wasm-c-api-table coverity-scan: if: ${{ github.repository == 'Samsung/walrus' && github.event_name == 'push' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install Packages - run: | - sudo apt update - sudo apt install -y ninja-build gcc-multilib g++-multilib - - name: Download Coverity Tool - env: - TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} - run: | - wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=$TOKEN&project=Samsung-walrus" -O cov-analysis-linux64.tar.gz - mkdir cov-analysis-linux64 - tar xzf cov-analysis-linux64.tar.gz --strip 1 -C cov-analysis-linux64 - - name: Build - env: - BUILD_OPTIONS: -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja - run: | - export PATH=$GITHUB_WORKSPACE/cov-analysis-linux64/bin:$PATH - cmake -H. -Bout/coverity_scan $BUILD_OPTIONS - cov-build --dir cov-int ninja -Cout/coverity_scan - - name: Submit - env: - TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} - NOTI_MAIL: ${{ secrets.COVERITY_SCAN_MAIL }} - run: | - tar czvf walrus.tgz cov-int - curl \ - --form token=$TOKEN \ - --form email=$NOTI_MAIL \ - --form file=@walrus.tgz \ - --form version="0.0.1" \ - --form description="walrus coverity scan" \ - https://scan.coverity.com/builds?project=Samsung-walrus + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Packages + run: | + sudo apt update + sudo apt install -y ninja-build gcc-multilib g++-multilib + - name: Download Coverity Tool + env: + TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + run: | + wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=$TOKEN&project=Samsung-walrus" -O cov-analysis-linux64.tar.gz + mkdir cov-analysis-linux64 + tar xzf cov-analysis-linux64.tar.gz --strip 1 -C cov-analysis-linux64 + - name: Build + env: + BUILD_OPTIONS: -DWALRUS_MODE=release -DWALRUS_OUTPUT=shell -GNinja + run: | + export PATH=$GITHUB_WORKSPACE/cov-analysis-linux64/bin:$PATH + cmake -H. -Bout/coverity_scan $BUILD_OPTIONS + cov-build --dir cov-int ninja -Cout/coverity_scan + - name: Submit + env: + TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + NOTI_MAIL: ${{ secrets.COVERITY_SCAN_MAIL }} + run: | + tar czvf walrus.tgz cov-int + curl \ + --form token=$TOKEN \ + --form email=$NOTI_MAIL \ + --form file=@walrus.tgz \ + --form version="0.0.1" \ + --form description="walrus coverity scan" \ + https://scan.coverity.com/builds?project=Samsung-walrus diff --git a/test/wasmBenchmarker/benchmark.py b/test/wasmBenchmarker/benchmark.py index a2934fa52..10eeb143c 100755 --- a/test/wasmBenchmarker/benchmark.py +++ b/test/wasmBenchmarker/benchmark.py @@ -115,17 +115,17 @@ def parse_args(): return args engine_path = 0 -def engine_display_name(engine): +def engine_display_name(engine): if not engine in engine_map: if engine_path == -2: engine_map[engine] = str(len(engine_map)) - + if engine_path == -1: engine_map[engine] = "" - + if engine_path == 0: engine_map[engine] = engine - + if engine_path > 0: engine_map[engine] = "/".join(engine.split("/")[0-engine_path:]) @@ -143,7 +143,7 @@ def check_programs(engines, verbose): def get_emcc(verbose, system_emcc=True): - emcc_path = None + emcc_path = None if system_emcc and os.system("emcc --version >/dev/null") == 0: if (verbose): print("Emscripten already installed on system") @@ -319,7 +319,7 @@ def run_tests(path, test_names, engines, number_of_runs, mem, no_time, jit, jit_ if sum(time_results[engine["name"]]) < 0: record[engine["name"]] = -1 continue - try: + try: value = (sum(time_results[engine["name"]]) / len(time_results[engine["name"]])) / 1e9 except ZeroDivisionError: value = -1 @@ -332,7 +332,7 @@ def run_tests(path, test_names, engines, number_of_runs, mem, no_time, jit, jit_ if sum(mem_results[engine["name"]]) < 0: record[engine["name"]] = -1 continue - try: + try: value = (sum(mem_results[engine["name"]]) / len(mem_results[engine["name"]])) except ZeroDivisionError: value = -1 @@ -345,7 +345,7 @@ def run_tests(path, test_names, engines, number_of_runs, mem, no_time, jit, jit_ def generate_report(data, summary, file_name=None): if summary: df = pd.DataFrame.from_records(data) - for col in df.columns: + for col in df.columns: if col == "test": continue @@ -353,7 +353,7 @@ def generate_report(data, summary, file_name=None): df[col] = df[col].str.split(' ').str[-1].str[1:-2] df[col] = df[col].astype(float) - + df = df.describe().loc[["mean"]].to_dict('records') df[0]["test"] = "MEAN" separator = [{}] @@ -368,7 +368,7 @@ def generate_report(data, summary, file_name=None): print("\n\n# Engines") for engine, serial in engine_map.items(): print(f"{serial}: {engine}") - + return with open(file_name, "w") as file: if file_name.endswith(".csv"): @@ -388,7 +388,7 @@ def generate_report(data, summary, file_name=None): line += "\n" file.write(line) - + if engine_path == -2: file.write("\n\n# Engines\n") for engine, serial in engine_map.items(): @@ -415,7 +415,7 @@ def compare(data, engines, jit_to_interpreter, jit_no_reg_alloc_to_interpreter, jit_no_reg_alloc = data[i][f"{engine_display_name(engine)} JIT_NO_REG_ALLOC"] interpreter = data[i][f"{engine_display_name(engine)} INTERPRETER"] data[i][f"{engine_display_name(engine)} INTERPRETER/JIT_NO_REG_ALLOC"] = f"{jit_no_reg_alloc} ({'{:.2f}'.format(-1 if float(interpreter) < 0 or float(jit_no_reg_alloc) < 0 else float(interpreter) / float(jit_no_reg_alloc))}x)" - + if jit_no_reg_alloc_to_jit: jit_no_reg_alloc = data[i][f"{engine_display_name(engine)} JIT_NO_REG_ALLOC"] jit = data[i][f"{engine_display_name(engine)} JIT"] @@ -430,7 +430,7 @@ def compare(data, engines, jit_to_interpreter, jit_no_reg_alloc_to_interpreter, interpreter = data[i][f"{engine_display_name(engine)} INTERPRETER"] jit_no_reg_alloc = data[i][f"{engine_display_name(engine)} JIT_NO_REG_ALLOC"] data[i][f"{engine_display_name(engine)} JIT_NO_REG_ALLOC/INTERPRETER"] = f"{interpreter} ({'{:.2f}'.format(-1 if float(jit_no_reg_alloc) < 0 or float(interpreter) < 0 else float(jit_no_reg_alloc) / float(interpreter))}x)" - + if jit_to_jit_no_reg_alloc: jit = data[i][f"{engine_display_name(engine)} JIT"] jit_no_reg_alloc = data[i][f"{engine_display_name(engine)} JIT_NO_REG_ALLOC"] @@ -499,6 +499,6 @@ def main(): if len(errorList) > 0: print(errorList) exit(1) - + if __name__ == "__main__": main() diff --git a/tools/run-tests.py b/tools/run-tests.py index b689449d2..ac4e0227c 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -28,7 +28,7 @@ from glob import glob from os.path import abspath, basename, dirname, join, relpath from shutil import copy -from subprocess import PIPE, Popen +from subprocess import PIPE, Popen, run, CalledProcessError PROJECT_SOURCE_DIR = dirname(dirname(abspath(__file__))) @@ -47,6 +47,7 @@ DEFAULT_RUNNERS = [] JIT_EXCLUDE_FILES = [] jit = False +jit_no_reg_alloc = False class runner(object): @@ -64,24 +65,35 @@ def __call__(self, fn): def _run_wast_tests(engine, files, is_fail, args=None): fails = 0 for file in files: - if jit: + if jit or jit_no_reg_alloc: filename = os.path.basename(file) if filename in JIT_EXCLUDE_FILES: continue - subprocess_args = [engine, "--mapdirs", "./test/wasi", "/var"] - if jit: subprocess_args.append("--jit") + subprocess_args = qemu + [engine, "--mapdirs", "./test/wasi", "/var"] + if jit or jit_no_reg_alloc: subprocess_args.append("--jit") + if jit_no_reg_alloc: subprocess_args.append("--jit-no-reg-alloc") if args: subprocess_args.append("--args") subprocess_args.append(file) if args: subprocess_args.extend(args) - proc = Popen(subprocess_args, stdout=PIPE) - out, _ = proc.communicate() - - if is_fail and proc.returncode or not is_fail and not proc.returncode: + if len(qemu) == 0: + proc = Popen(subprocess_args, stdout=PIPE, stderr=PIPE) + out, _ = proc.communicate() + returncode = proc.returncode + else: + try: + run(subprocess_args, check=True, stdout=PIPE) + returncode = 0 + out = "" + except CalledProcessError as e: + returncode = e.returncode + out = e.stdout.decode('utf-8') + + if is_fail and returncode or not is_fail and not returncode: print('%sOK: %s%s' % (COLOR_GREEN, file, COLOR_RESET)) else: - print('%sFAIL(%d): %s%s' % (COLOR_RED, proc.returncode, file, COLOR_RESET)) + print('%sFAIL(%d): %s%s' % (COLOR_RED, returncode, file, COLOR_RESET)) print(out) fails += 1 @@ -185,13 +197,25 @@ def main(): parser = ArgumentParser(description='Walrus Test Suite Runner') parser.add_argument('--engine', metavar='PATH', default=DEFAULT_WALRUS, help='path to the engine to be tested (default: %(default)s)') + parser.add_argument('--qemu', metavar='PATH', default=None, help='path to qemu') parser.add_argument('suite', metavar='SUITE', nargs='*', default=sorted(DEFAULT_RUNNERS), help='test suite to run (%s; default: %s)' % (', '.join(sorted(RUNNERS.keys())), ' '.join(sorted(DEFAULT_RUNNERS)))) parser.add_argument('--jit', action='store_true', help='test with JIT') + parser.add_argument('--jit-no-reg-alloc', action='store_true', help='test with JIT without register allocation') args = parser.parse_args() global jit jit = args.jit - if jit: + + global jit_no_reg_alloc + jit_no_reg_alloc = args.jit_no_reg_alloc + + global qemu + qemu = [args.qemu] if args.qemu else [] + + if jit and jit_no_reg_alloc: + parser.error('jit and jit-no-reg-alloc cannot be used together') + + if jit or jit_no_reg_alloc: exclude_list_file = join(PROJECT_SOURCE_DIR, 'tools', 'jit_exclude_list.txt') with open(exclude_list_file) as f: global JIT_EXCLUDE_FILES @@ -205,7 +229,12 @@ def main(): success, fail = [], [] for suite in args.suite: - print(COLOR_PURPLE + f'running test suite{ " with jit" if jit else ""}: ' + suite + COLOR_RESET) + text = "" + if jit: + text = " with jit" + elif jit_no_reg_alloc: + text = " with jit without register allocation" + print(COLOR_PURPLE + f'running test suite{text}: ' + suite + COLOR_RESET) try: RUNNERS[suite](args.engine) success += [suite]