Merge branch 'master' into faster-hashes

Author: advaita-saha

.github/workflows/ci.yml

@@ -22,6 +22,7 @@ on:
       - '.github/workflows/nimbus_verified_proxy.yml'
       - '.github/workflows/build_base_image.yml'
       - 'docker/**'
+      - 'metrics/**'
 
   pull_request:
     paths-ignore:
@@ -33,6 +34,7 @@ on:
       - '.github/workflows/nimbus_verified_proxy.yml'
       - '.github/workflows/build_base_image.yml'
       - 'docker/**'
+      - 'metrics/**'
 
   workflow_dispatch:
 

.github/workflows/kurtosis.yml

@@ -19,21 +19,19 @@ on:
       - 'portal/docs/**'
       - '**/*.md'
       - 'hive_integration/**'
-      - 'nimbus_verified_proxy/**'
-      - '.github/workflows/nimbus_verified_proxy.yml'
       - '.github/workflows/build_base_image.yml'
       - 'docker/**'
+      - 'metrics/**'
 
   pull_request:
     paths-ignore:
       - 'doc/**'
       - 'portal/docs/**'
       - '**/*.md'
       - 'hive_integration/**'
-      - 'nimbus_verified_proxy/**'
-      - '.github/workflows/nimbus_verified_proxy.yml'
       - '.github/workflows/build_base_image.yml'
       - 'docker/**'
+      - 'metrics/**'
 
 concurrency: # Cancel stale PR builds (but not push builds)
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}

.github/workflows/nimbus_verified_proxy.yml

@@ -13,26 +13,30 @@ on:
   push:
     branches:
       - master
-    paths:
-      - '.github/workflows/nimbus_verified_proxy.yml'
-      - 'nimbus_verified_proxy/**'
-      - '!nimbus_verified_proxy/**.md'
-      - '!nimbus_verified_proxy/docs/**'
-      - 'execution_chain/db/**'
-      - 'vendor/**'
-      - 'Makefile'
-      - 'nimbus.nimble'
+    paths-ignore:
+      - 'doc/**'
+      - 'portal/docs/**'
+      - '**/*.md'
+      - 'portal/**'
+      - '.github/workflows/portal.yml'
+      - '.github/workflows/portal_docs.yml'
+      - '.github/workflows/build_base_image.yml'
+      - 'docker/**'
+      - 'metrics/**'
 
   pull_request:
-    paths:
-      - '.github/workflows/nimbus_verified_proxy.yml'
-      - 'nimbus_verified_proxy/**'
-      - '!nimbus_verified_proxy/**.md'
-      - '!nimbus_verified_proxy/docs/**'
-      - 'execution_chain/db/**'
-      - 'vendor/**'
-      - 'Makefile'
-      - 'nimbus.nimble'
+    paths-ignore:
+      - 'doc/**'
+      - 'portal/docs/**'
+      - '**/*.md'
+      - 'portal/**'
+      - '.github/workflows/portal.yml'
+      - '.github/workflows/portal_docs.yml'
+      - '.github/workflows/build_base_image.yml'
+      - 'docker/**'
+      - 'metrics/**'
+
+  workflow_dispatch:
 
 concurrency: # Cancel stale PR builds (but not push builds)
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -64,31 +68,12 @@ jobs:
         with:
           llvm-mingw: true
           nim-cache: true
-          rocksdb-cache: true
+          rocksdb-cache: false
           eest-cache: false
 
-      - name: Run verified proxy tests (Windows)
-        if: runner.os == 'Windows'
+      - name: Run verified proxy tests
         run: |
           gcc --version
-          DEFAULT_MAKE_FLAGS="-j${ncpu}"
-          mingw32-make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy libverifproxy
-          build/nimbus_verified_proxy.exe --help
-          mingw32-make ${DEFAULT_MAKE_FLAGS} nimbus-verified-proxy-test
-
-      - name: Run verified proxy tests (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
-          DEFAULT_MAKE_FLAGS="-j${ncpu}"
-          env CC=gcc make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy libverifproxy
-          build/nimbus_verified_proxy --help
-          # CC is needed to select correct compiler 32/64 bit
-          env CC=gcc CXX=g++ make ${DEFAULT_MAKE_FLAGS} nimbus-verified-proxy-test
-
-      - name: Run verified proxy tests (Macos)
-        if: runner.os == 'Macos'
-        run: |
           DEFAULT_MAKE_FLAGS="-j${ncpu}"
           make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy libverifproxy
           build/nimbus_verified_proxy --help

.github/workflows/portal.yml

@@ -18,6 +18,7 @@ on:
       - '.github/workflows/nimbus_verified_proxy.yml'
       - '.github/workflows/build_base_image.yml'
       - 'docker/**'
+      - 'metrics/**'
 
   pull_request:
     paths-ignore:
@@ -28,6 +29,7 @@ on:
       - '.github/workflows/nimbus_verified_proxy.yml'
       - '.github/workflows/build_base_image.yml'
       - 'docker/**'
+      - 'metrics/**'
 
   workflow_dispatch:
 
@@ -132,14 +134,14 @@ jobs:
         with:
           llvm-mingw: true
           nim-cache: true
-          rocksdb-cache: true
+          rocksdb-cache: false
           eest-cache: false
 
       - name: Run Nimbus Portal tests (Windows)
         if: runner.os == 'Windows'
         run: |
           gcc --version
-          DEFAULT_MAKE_FLAGS="-j${ncpu} ROCKSDB_CI_CACHE=RocksBinCache"
+          DEFAULT_MAKE_FLAGS="-j${ncpu}"
           mingw32-make ${DEFAULT_MAKE_FLAGS} nimbus_portal_client
           build/nimbus_portal_client.exe --help
           # find . -type d -name ".git" -exec rm -rf {} +
@@ -156,7 +158,7 @@ jobs:
           ./env.sh nim -v
           ldd --version
           export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
-          DEFAULT_MAKE_FLAGS="-j${ncpu} ROCKSDB_CI_CACHE=RocksBinCache"
+          DEFAULT_MAKE_FLAGS="-j${ncpu}"
           env CC=gcc make ${DEFAULT_MAKE_FLAGS} nimbus_portal_client
           build/nimbus_portal_client --help
           env CC=gcc make ${DEFAULT_MAKE_FLAGS} portal-tools
@@ -166,7 +168,7 @@ jobs:
       - name: Run Nimbus Portal tests (Macos)
         if: runner.os == 'Macos'
         run: |
-          DEFAULT_MAKE_FLAGS="-j${ncpu} ROCKSDB_CI_CACHE=RocksBinCache"
+          DEFAULT_MAKE_FLAGS="-j${ncpu}"
           make ${DEFAULT_MAKE_FLAGS} nimbus_portal_client
           build/nimbus_portal_client --help
           make ${DEFAULT_MAKE_FLAGS} portal-tools

.gitignore

@@ -51,3 +51,5 @@ tests/fixtures/eest_static
 tests/fixtures/eest_stable
 tests/fixtures/eest_develop
 tests/fixtures/eest_devnet
+
+execution_chain/nimbus

.gitmodules

@@ -229,3 +229,7 @@
 [submodule "vendor/constantine"]
 	path = vendor/constantine
 	url = https://github.com/mratsim/constantine
+[submodule "vendor/nim-mcl"]
+	path = vendor/nim-mcl
+	url = https://github.com/status-im/nim-mcl
+  branch = main

GeneralStateTestsDevnet.md

@@ -139,6 +139,11 @@ OK: 5/5 Fail: 0/5 Skip: 0/5
 + valid_multi_inf.json                                            OK
 ```
 OK: 6/6 Fail: 0/6 Skip: 0/6
+## bls12_precompiles_before_fork
+```diff
++ precompile_before_fork.json                                     OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
 ## bls12_variable_length_input_contracts
 ```diff
 + invalid_gas_g1msm.json                                          OK
@@ -165,6 +170,11 @@ OK: 15/15 Fail: 0/15 Skip: 0/15
 + call_memory_expands_on_early_revert.json                        OK
 ```
 OK: 3/3 Fail: 0/3 Skip: 0/3
+## call_and_callcode_gas_calculation
+```diff
++ value_transfer_gas_calculation.json                             OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
 ## calldatacopy
 ```diff
 + calldatacopy.json                                               OK
@@ -461,9 +471,10 @@ OK: 50/50 Fail: 0/50 Skip: 0/50
 + pointer_to_precompile.json                                      OK
 + pointer_to_static.json                                          OK
 + pointer_to_static_reentry.json                                  OK
++ set_code_type_tx_pre_fork.json                                  OK
 + static_to_pointer.json                                          OK
 ```
-OK: 15/15 Fail: 0/15 Skip: 0/15
+OK: 16/16 Fail: 0/16 Skip: 0/16
 ## shift_combinations
 ```diff
 + combinations.json                                               OK
@@ -551,4 +562,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 1/1 Fail: 0/1 Skip: 0/1
 
 ---TOTAL---
-OK: 265/265 Fail: 0/265 Skip: 0/265
+OK: 268/268 Fail: 0/268 Skip: 0/268

Makefile

@@ -131,16 +131,13 @@ VERIF_PROXY_OUT_PATH ?= build/libverifproxy/
 ifeq ($(NIM_PARAMS),)
 # "variables.mk" was not included, so we update the submodules.
 # selectively download nimbus-eth2 submodules because we don't need all of it's modules
-# also holesky already exceeds github LFS quota
+# also hoodi already exceeds github LFS quota
 
-# We don't need these `vendor/holesky` files but fetching them
-# may trigger 'This repository is over its data quota' from GitHub
 GIT_SUBMODULE_CONFIG := -c lfs.fetchexclude=/public-keys/all.txt,/metadata/genesis.ssz,/parsed/parsedConsensusGenesis.json
 
 GIT_SUBMODULE_UPDATE := git -c submodule."vendor/nimbus-eth2".update=none submodule update --init --recursive; \
   git $(GIT_SUBMODULE_CONFIG) submodule update vendor/nimbus-eth2; \
   cd vendor/nimbus-eth2; \
-  git $(GIT_SUBMODULE_CONFIG) submodule update --init vendor/holesky; \
   git $(GIT_SUBMODULE_CONFIG) submodule update --init vendor/sepolia; \
   git $(GIT_SUBMODULE_CONFIG) submodule update --init vendor/hoodi; \
   git $(GIT_SUBMODULE_CONFIG) submodule update --init vendor/gnosis-chain-configs; \
@@ -161,7 +158,7 @@ GIT_SUBMODULE_UPDATE := git -c submodule."vendor/nimbus-eth2".update=none submod
 else # "variables.mk" was included. Business as usual until the end of this file.
 
 # default target, because it's the first one that doesn't start with '.'
-all: | $(TOOLS) nimbus_execution_client
+all: | $(TOOLS) nimbus nimbus_execution_client
 
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
@@ -218,8 +215,9 @@ nimbus_execution_client: | build deps rocksdb
 check_revision: nimbus_execution_client
 	scripts/check_revision.sh
 
-nimbus: nimbus_execution_client
-	echo "The nimbus target is deprecated and will soon change meaning, use 'nimbus_execution_client' instead"
+nimbus: | build deps rocksdb
+	echo -e $(BUILD_MSG) "build/nimbus" && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/nimbus "execution_chain/nimbus.nim"
 
 # symlink
 nimbus.nims:
@@ -290,20 +288,16 @@ portal-test-reproducibility:
 		[ "$$MD5SUM1" = "$$MD5SUM2" ] && echo -e "\e[92mSuccess: identical binaries.\e[39m" || \
 			{ echo -e "\e[91mFailure: the binary changed between builds.\e[39m"; exit 1; }
 
-# Portal tests
-all_eth_history_custom_chain_tests: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -d:mergeBlockNumber:38130 -o:build/$@ "portal/tests/eth_history_tests/$@.nim"
-
+# builds and runs the Portal test suite
 all_portal_tests: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -o:build/$@ "portal/tests/$@.nim"
 
-# builds and runs the Portal test suite
-portal-test: | all_portal_tests all_eth_history_custom_chain_tests
+# alias for all_portal_tests
+portal-test: | all_portal_tests
 
 # builds the Portal tools, wherever they are
-$(PORTAL_TOOLS): | build deps rocksdb
+$(PORTAL_TOOLS): | build deps
 	for D in $(PORTAL_TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \
 		echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$@ "$${TOOL_DIR}/$@.nim"
@@ -333,7 +327,7 @@ fluffy: | build deps
 		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$@ "portal/client/nimbus_portal_client.nim"
 
 # Legacy target, same as nimbus_portal_bridge, deprecated
-portal_bridge: | build deps rocksdb
+portal_bridge: | build deps
 	echo -e "\033[0;31mWarning:\033[0m The portal_bridge target and binary is deprecated, use 'make nimbus_portal_bridge' instead"
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$@ "portal/bridge/nimbus_portal_bridge.nim"

PrecompileTests.md

@@ -28,6 +28,7 @@ PrecompileTests
 OK: 22/22 Fail: 0/22 Skip: 0/22
 ## eest
 ```diff
++ P256Verify.json                                                 OK
 + add_G1_bls.json                                                 OK
 + add_G2_bls.json                                                 OK
 + fail-add_G1_bls.json                                            OK
@@ -47,7 +48,7 @@ OK: 22/22 Fail: 0/22 Skip: 0/22
 + mul_G2_bls.json                                                 OK
 + pairing_check_bls.json                                          OK
 ```
-OK: 18/18 Fail: 0/18 Skip: 0/18
+OK: 19/19 Fail: 0/19 Skip: 0/19
 
 ---TOTAL---
-OK: 40/40 Fail: 0/40 Skip: 0/40
+OK: 41/41 Fail: 0/41 Skip: 0/41

README.md

@@ -310,7 +310,7 @@ make nimbus_execution_client
 # the Prometheus daemon will create its data dir in the current dir, so give it its own directory
 mkdir ../my_metrics
 # copy the basic config file over there
-cp -a examples/prometheus.yml ../my_metrics/
+cp -a metrics/prometheus.yml ../my_metrics/
 # start Prometheus in a separate terminal
 cd ../my_metrics
 prometheus --config.file=prometheus.yml # loads ./prometheus.yml, writes metric data to ./data
@@ -327,7 +327,7 @@ the form field, unless you click on it.
 
 Create a new dashboard. Click on its default title in the upper left corner
 ("New Dashboard"). In the new page, click "Import dashboard" in the right
-column and upload "examples/Nimbus-Grafana-dashboard.json".
+column and upload "metrics/grafana/grafana.infra.status.im.json".
 
 In the main panel, there's a hidden button used to assign metrics to the left
 or right Y-axis - it's the coloured line on the left of the metric name, in the