Move benchmark contracts to era_vm (#218)

* Add benchmark contracts and modify Makefile accordingly * Add benchmark results processing script * Update zksync-era commit pointer * Add flamegraph command to Makefile * makefile: make contract build always phony * makefile: update target name * makefile: update makefile * ci: add flamegraph check * ci: cargo-flamegraph through binstall * ci: install flamegraph with cargo-bin * ci: add zksync-home env-var * ci: add perf deps * ci: do not open svg on fg check * ci: do not check flamegraphs --------- Co-authored-by: Julian Ventura <[email protected]> Co-authored-by: Fran <[email protected]>
lambdaclass · Aug 29, 2024 · 6b5efb9 · 6b5efb9
1 parent 817d0f3
commit 6b5efb9
Show file tree

Hide file tree

Showing 10 changed files with 194 additions and 10 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -552,10 +552,10 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v4
 
-      - name: Zksync-era dependencies
+      - name: Zksync-era + perf dependencies
         uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          packages: clang clang-tools build-essential librocksdb-dev
+          packages: clang clang-tools build-essential librocksdb-dev  linux-tools-common linux-tools-generic linux-perf
           version: 1.0
 
       - name: Setup nodejs + yarn
@@ -585,25 +585,31 @@ jobs:
           chmod +x solc
           sudo cp solc /usr/bin/solc
 
-      - name: Build benchmark contracts
-        working-directory: ${{ github.workspace }}
-        run: make bench-setup
 
-      - name: Fetch toolchain version from zksync-era
+      - name: Fetch toolchain version from zksync-era + set zksync home
         run: |
           cd ${{ github.workspace }}/zksync-era
           echo "ERA_TOOLCHAIN=$(head ./rust-toolchain)" >> $GITHUB_ENV
+          echo "ZKSYNC_HOME=${{ github.workspace }}/zksync-era" >> $GITHUB_ENV
 
       - name: Rustup toolchain install
         uses: dtolnay/rust-toolchain@nightly
         with:
           toolchain: ${{ env.ERA_TOOLCHAIN }}
 
+      - name: Build benchmark contracts
+        working-directory: ${{ github.workspace }}
+        run: make bench-setup
+
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: |
             ${{ github.workspace }}/zksync-era/core/tests/vm-benchmark
 
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: flamegraph
+
       - name: Check zksync-era benchmarks & era_vm build correctly
         run: |
           cd ${{ github.workspace }}/zksync-era

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: clean lint test deps submodules bench era-test build-bench-contracts %.sol
+.PHONY: clean lint test deps submodules bench flamegraph era-test build-bench-contracts %.sol
 .SILENT: %.sol
 
 LLVM_PATH?=$(shell pwd)/era-compiler-tester/target-llvm/target-final/
@@ -9,7 +9,7 @@ ZKSYNC_SYS_CONTRACTS=$(ZKSYNC_ROOT)/contracts/system-contracts/artifacts-zk
 ZKSYNC_BOOTLOADER_CONTRACT=$(ZKSYNC_ROOT)/contracts/system-contracts/bootloader/build/artifacts
 ZKSYNC_BENCH_TEST_DATA=$(ZKSYNC_ROOT)/etc/contracts-test-data/artifacts-zk
 ZKSYNC_BENCH_CONTRACTS=$(ZKSYNC_ROOT)/core/tests/vm-benchmark/deployment_benchmarks
-ZKSYNC_BENCH_SOURCES=$(ZKSYNC_ROOT)/core/tests/vm-benchmark/deployment_benchmarks_sources
+BENCH_SOURCES=$(shell realpath ./deployment_benchmarks_sources)
 
 
 clean:
@@ -71,7 +71,7 @@ $(ZKSYNC_BENCH_TEST_DATA):
 #     a file insie the contract benchmarks folder.
 %.sol:
 	echo "Building benchmark contract: $@"
-	cd $(ZKSYNC_BENCH_SOURCES) && \
+	cd $(BENCH_SOURCES) && \
 	zksolc --bin $@ | grep -oE '0x[0-9a-fA-F]+' > $(ZKSYNC_BENCH_CONTRACTS)/$(basename $@)
 
 build_bench_contracts: fibonacci_rec.sol send.sol
@@ -84,6 +84,9 @@ bench-setup: submodules build_bench_contracts $(ZKSYNC_BENCH_TEST_DATA) $(ZKSYNC
 bench:
 	cd $(ZKSYNC_ROOT) && cargo bench --bench criterion "$(lambda|fast_vm|legacy)/(fibonacci|send)"
 
+check-flamegraph:
+	cd $(ZKSYNC_ROOT) && CARGO_PROFILE_BENCH_DEBUG=2 cargo flamegraph --root --bench criterion -- --bench --profile-time 5 "$lambda/fibonacci_rec^"
+
 bench-base:
 	cd $(ZKSYNC_ROOT) && cargo bench --bench criterion -- --save-baseline bench_base lambda 1>bench-compare.txt
 

diff --git a/bench_results.py b/bench_results.py
@@ -0,0 +1,55 @@
+import re
+import sys
+
+def convert_to_ms(value, unit):
+    if unit == 'µs':
+        return value / 1000
+    elif unit == 'ms':
+        return value
+    elif unit == 's':
+        return value * 1000
+    else:
+        raise ValueError(f"Unknown time unit: {unit}")
+
+def parse_benchmark_file(file_path):
+    machines = {}
+    pattern = r'(\w+)/\w+\s+time:\s+\[\d+(?:\.\d+)?\s(\w+)\s(\d+(?:\.\d+)?)\s(\w+)\s\d+(?:\.\d+)?\s(\w+)\]'
+
+    with open(file_path, 'r') as file:
+        content = file.read()
+        # Clean the file content
+        cleaned_content = re.sub(r'\n(\s+)time:', ' time:', content)
+        for line in cleaned_content.split('\n'):
+            match = re.match(pattern, line)
+            if match:
+                machine, lower_unit, mid_time, mid_unit, upper_unit = match.groups()
+                # Convert mid_time to milliseconds
+                mid_time_ms = convert_to_ms(float(mid_time), mid_unit)
+                machines[machine] = machines.get(machine, 0) + mid_time_ms
+
+    return machines
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: results.py <path-to-file>")
+        return
+    file_path = sys.argv[1]
+    results = parse_benchmark_file(file_path)
+
+    for machine, total_time in results.items():
+        print(f"Total {machine} time: {total_time:.3f} ms")
+
+    if not "lambda" in results:
+        return
+
+    print("")
+
+    if "legacy" in results:
+        lambda_vs_legacy = round(results["lambda"] / results["legacy"], 1)
+        print(f"lambda_vm took x{lambda_vs_legacy} more than legacy_vm")
+    if "fast" in results:
+        lambda_vs_fast = round(results["lambda"] / results["fast"], 1)
+        print(f"lambda_vm took x{lambda_vs_fast} more than fast_vm")
+
+if __name__ == "__main__":
+    main()
diff --git a/deployment_benchmarks_sources/factorial_it.sol b/deployment_benchmarks_sources/factorial_it.sol
@@ -0,0 +1,22 @@
+pragma solidity ^0.8.0;
+
+contract benchmark {
+    uint256 value;
+    constructor() {
+      value = fac(57);
+    }
+
+    function get_calculation() public view returns (uint256) {
+        return value;
+    }
+
+    function fac(uint n) internal pure returns(uint256) { 
+        uint256 r = 1;
+        for (uint256 k = 2; k <= n; k++) {
+            r = k * r;
+        }
+
+        return r;
+    }
+}
+
diff --git a/deployment_benchmarks_sources/factorial_rec.sol b/deployment_benchmarks_sources/factorial_rec.sol
@@ -0,0 +1,20 @@
+pragma solidity ^0.8.0;
+
+contract benchmark {
+    uint256 value;
+    constructor() {
+      value = fac(57);
+    }
+
+    function get_calculation() public view returns (uint256) {
+        return value;
+    }
+
+    function fac(uint256 n) internal returns (uint256) {
+        if (n <= 1) {
+            return 1;
+        }
+        return n * fac(n - 1);
+    }
+}
+
diff --git a/deployment_benchmarks_sources/fibonacci_it.sol b/deployment_benchmarks_sources/fibonacci_it.sol
@@ -0,0 +1,26 @@
+pragma solidity ^0.8.0;
+
+contract benchmark {
+    uint256 value;
+    constructor() {
+      value = fib(370);
+    }
+
+    function get_calculation() public view returns (uint256) {
+        return value;
+    }
+
+    function fib(uint n) internal pure returns(uint b) { 
+        if (n == 0) {
+            return 0;   
+        }
+        uint a = 1;
+        b = 1;
+        for (uint i = 2; i < n; i++) {
+            uint c = a + b;
+            a = b;
+            b = c;
+        }
+        return b;
+    }
+}
diff --git a/deployment_benchmarks_sources/fibonacci_rec.sol b/deployment_benchmarks_sources/fibonacci_rec.sol
@@ -0,0 +1,19 @@
+pragma solidity ^0.8.0;
+
+contract benchmark {
+    uint256 value;
+    constructor() {
+      value = fib(25);
+    }
+
+    function get_calculation() public view returns (uint256) {
+        return value;
+    }
+
+    function fib(uint256 n) internal returns (uint256) {
+        if (n <= 1) {
+            return n;
+        }
+        return fib(n - 1) + fib(n - 2);
+    }
+}
diff --git a/deployment_benchmarks_sources/heap_read_write.sol b/deployment_benchmarks_sources/heap_read_write.sol
@@ -0,0 +1,25 @@
+pragma solidity ^0.8.0;
+
+contract HeapBenchmark {
+    constructor() {
+        uint256 v1 = 0;
+        uint256 v2 = 0;
+        uint256 n = 16000;
+        uint256[] memory array = new uint256[](1);
+
+        assembly {
+            mstore(add(array, sub(n, 1)), 4242)
+
+            let j := 0
+            for {} lt(j, n) {} {
+                v1 := mload(add(array, mod(mul(j, j), n)))
+                v2 := mload(add(array, j))
+                mstore(add(array, j), add(add(v1, v2), 42))
+                j := add(j, 1) 
+                if gt(j, sub(n, 1)) {
+                    j := 0
+                }
+            }
+        }
+    }
+}
diff --git a/deployment_benchmarks_sources/send.sol b/deployment_benchmarks_sources/send.sol
@@ -0,0 +1,8 @@
+pragma solidity >=0.8.20;
+
+contract Send {
+    constructor() {
+        address payable self = payable(msg.sender);
+        self.call{value: 100};
+    }
+}
diff --git a/zksync-era b/zksync-era