Unified Nimbus node

This change brings Nimbus full circle to where it started all these
years ago and allows running Ethereum in a single node / process, both
as a wallet/web3 backend and as a validator.

Among interesting properies are:

* Easy to set up and run - one binary, one process, no JWT and messy
  setup, no cross-client communication issues, timing issues etc
* Excellent performance, of course
* Shared database = small database
* Can run without the legacy devp2p as long as you're reasonably synced
  and not using it for block production - up to 5 months of history are
  instead sourced from the consensus network - block production requires
  devp2p since that's where the public mempool comes from

Running ethereum and syncing mainnet is now as easy as:

```sh
./nimbus trustedNodeSync \
  --trusted-node-url=http://testing.mainnet.beacon-api.nimbus.team/ \
  --backfill=false

./nimbus
```

The consensus chain will start from a checkpoint while the execution
chain will still be synced via P2P.

You need about 500GB of space in total, but if you're buying a drive
today, get 2 or 4 TB anyway.

Testnets like `hoodi` can reasonably be synced from P2P all the way
(takes a bit more than a day at the time of writing), without the
checkpoint sync:

```nim
./nimbus --network:hoodi
```

That's it! The node can now be used both for validators and as a web3
provider. `--rpc` gives you the web3 backend which allows connecting
wallets while `--rest` gives the beacon api that validator clients use.

Of course, you can run your validators [in the
node](https://nimbus.guide/run-a-validator.html#2-import-your-validator-keys)
as well.

Here's a true maxi configuration that turns on (almost) everything:

```nim
./nimbus --rpc --rest --metrics
```

The execution chain can also be imported from era files, downloading
the history from https://mainnet.era.nimbus.team/ and
https://mainnet.era1.nimbus.team/ and placing them in `era` and `era1`
in the data directory as the
[manual](https://nimbus.guide/execution-client.html#syncing-using-era-files)
suggests, then running an `import` - it takes a few days:

```sh
./nimbus import
```

If you were already running nimbus, you can reuse your existing data
directory - use `--data-dir:/some/path` as usual with all the commands
to specify where you want your data stored - if you had both eth1 and
eth2 directories, just merge their contents.

To get up and running more quickly, snapshots of the mainnet execution
database are maintained here:

https://eth1-db.nimbus.team/

Together with checkpoint sync, you'll have a fully synced node in no
time!

In future versions, this will be replaced by snap sync or an equivalent
state sync mechanism.

To build the protoype:

```sh
make update
make -j8 nimbus
```

In a single process binary, the beacon and execution chain are each
running in their own thread, sharing data directory and common services,
similar to running the two pieces separately with the same data dir.

One way to think about it is that the execution client and beacon nodes
are stand-alone libraries that are being used together - this is not far
from the truth and in fact, you can use either (or both!) as a library.

The binary supports the union of all functionality that
`nimbus_execution_client` and `nimbus_beacon_node` offers, including all
the subcommands like [checkpoint
sync](https://nimbus.guide/trusted-node-sync.html)
and [execution history
import](https://nimbus.guide/execution-client.html#import-era-files),
simply using the `nimbus` command instead.

Prototype notes:

* cross-thread communication is done using a local instance of web3 /
  JSON - this is nuts of course: it should simply pass objects around
  and convert to directly to RLP on demand without going via JSON
* the thread pool is not shared but should be - nim-taskpools needs to
  learn to accept tasks from threads other than the one that created it
* discovery is not shared - instead, each of eth1/2 runs its own
  discovery protocols and consequently the node has two "identities"
* there are many efficiency opportunities to exploit, in particular on
  the memory usage front
* next up is light client and portal to be added as options, to support
  a wide range of feature vs performance tradeoffs

Author: arnetheduck

.gitignore

@@ -51,3 +51,5 @@ tests/fixtures/eest_static
 tests/fixtures/eest_stable
 tests/fixtures/eest_develop
 tests/fixtures/eest_devnet
+
+execution_chain/nimbus

Makefile

@@ -218,8 +218,9 @@ nimbus_execution_client: | build deps rocksdb
 check_revision: nimbus_execution_client
 	scripts/check_revision.sh
 
-nimbus: nimbus_execution_client
-	echo "The nimbus target is deprecated and will soon change meaning, use 'nimbus_execution_client' instead"
+nimbus: | build deps rocksdb
+	echo -e $(BUILD_MSG) "build/nimbus" && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/nimbus "execution_chain/nimbus.nim"
 
 # symlink
 nimbus.nims:

execution_chain/beacon/api_handler/api_newpayload.nim

@@ -178,7 +178,7 @@ proc newPayload*(ben: BeaconEngineRef,
   # If we already have the block locally, ignore the entire execution and just
   # return a fake success.
   if chain.haveBlockAndState(blockHash):
-    notice "Ignoring already known beacon payload",
+    debug "Ignoring already known beacon payload",
       number = header.number, hash = blockHash.short
     return validStatus(blockHash)
 

execution_chain/conf.nim

@@ -37,7 +37,7 @@ import
   ./common/chain_config,
   ./db/opts
 
-export net, defs, jsdefs, jsnet, nimbus_binary_common, options
+export net, defs, jsdefs, jsnet, nat_toml, nimbus_binary_common, options
 
 const NimbusCopyright* =
   "Copyright (c) 2018-" & compileYear & " Status Research & Development GmbH"
@@ -51,7 +51,7 @@ func getLogLevels(): string =
   join(logLevels, ", ")
 
 const
-  defaultPort              = 30303
+  defaultExecutionPort*    = 30303
   defaultMetricsServerPort = 9093
   defaultHttpPort          = 8545
   # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/authentication.md#jwt-specifications
@@ -66,7 +66,7 @@ let
 
 type
   NimbusCmd* {.pure.} = enum
-    noCommand
+    executionClient
     `import`
     `import-rlp`
 
@@ -81,38 +81,39 @@ type
     V5
 
   ExecutionClientConf* = object
-    ## Main Nimbus configuration object
-    configFile {.
+    ## Main configuration for the execution client - when updating, coordinate
+    ## options shared with other executables (logging, metrics etc)
+    configFile* {.
       separator: "ETHEREUM OPTIONS:"
       desc: "Loads the configuration from a TOML file"
       name: "config-file" .}: Option[InputFile]
 
     dataDirFlag* {.
       desc: "The directory where nimbus will store all blockchain data"
       abbr: "d"
-      name: "data-dir" }: Option[OutDir]
+      name: "data-dir" .}: Option[OutDir]
 
     era1DirFlag* {.
       desc: "Directory where era1 (pre-merge) archive can be found"
       defaultValueDesc: "<data-dir>/era1"
-      name: "era1-dir" }: Option[OutDir]
+      name: "era1-dir" .}: Option[OutDir]
 
     eraDirFlag* {.
       desc: "Directory where era (post-merge) archive can be found"
       defaultValueDesc: "<data-dir>/era"
-      name: "era-dir" }: Option[OutDir]
+      name: "era-dir" .}: Option[OutDir]
 
     keyStoreDirFlag* {.
       desc: "Load one or more keystore files from this directory"
       defaultValueDesc: "inside datadir"
       abbr: "k"
-      name: "key-store" }: Option[OutDir]
+      name: "key-store" .}: Option[OutDir]
 
     importKey* {.
       desc: "Import unencrypted 32 bytes hex private key from a file"
       defaultValue: ""
       abbr: "e"
-      name: "import-key" }: InputFile
+      name: "import-key" .}: InputFile
 
     trustedSetupFile* {.
       desc: "Load EIP-4844 trusted setup file"
@@ -146,32 +147,31 @@ type
       defaultValue: @[] # the default value is set in makeConfig
       defaultValueDesc: "mainnet(1)"
       abbr: "i"
-      name: "network" }: seq[string]
+      name: "network" .}: seq[string]
 
     customNetwork {.
       ignore
       desc: "Use custom genesis block for private Ethereum Network (as /path/to/genesis.json)"
       defaultValueDesc: ""
       abbr: "c"
-      name: "custom-network" }: Option[NetworkParams]
+      name: "custom-network" .}: Option[NetworkParams]
 
     networkId* {.
       ignore # this field is not processed by confutils
       defaultValue: MainNet # the defaultValue value is set by `makeConfig`
       defaultValueDesc: "MainNet"
-      name: "network-id"}: NetworkId
+      name: "network-id" .}: NetworkId
 
     networkParams* {.
       ignore # this field is not processed by confutils
       defaultValue: NetworkParams() # the defaultValue value is set by `makeConfig`
-      name: "network-params"}: NetworkParams
+      name: "network-params" .}: NetworkParams
 
     logLevel* {.
       separator: "\pLOGGING AND DEBUGGING OPTIONS:"
       desc: "Sets the log level for process and topics (" & logLevelDesc & ")"
       defaultValue: "INFO"
-      defaultValueDesc: "Info topic level logging"
-      name: "log-level" }: string
+      name: "log-level" .}: string
 
     logStdout* {.
       hidden
@@ -183,79 +183,78 @@ type
     metricsEnabled* {.
       desc: "Enable the built-in metrics HTTP server"
       defaultValue: false
-      name: "metrics" }: bool
+      name: "metrics" .}: bool
 
     metricsPort* {.
       desc: "Listening port of the built-in metrics HTTP server"
       defaultValue: defaultMetricsServerPort
       defaultValueDesc: $defaultMetricsServerPort
-      name: "metrics-port" }: Port
+      name: "metrics-port" .}: Port
 
     metricsAddress* {.
       desc: "Listening IP address of the built-in metrics HTTP server"
       defaultValue: defaultAdminListenAddress
       defaultValueDesc: $defaultAdminListenAddressDesc
-      name: "metrics-address" }: IpAddress
+      name: "metrics-address" .}: IpAddress
 
     bootstrapNodes {.
       separator: "\pNETWORKING OPTIONS:"
       desc: "Specifies one or more bootstrap nodes(ENR or enode URL) to use when connecting to the network"
       defaultValue: @[]
       defaultValueDesc: ""
       abbr: "b"
-      name: "bootstrap-node" }: seq[string]
+      name: "bootstrap-node" .}: seq[string]
 
     bootstrapFile {.
       desc: "Specifies a file of bootstrap Ethereum network addresses(ENR or enode URL). " &
             "Both line delimited or YAML format are supported"
       defaultValue: ""
-      name: "bootstrap-file" }: InputFile
+      name: "bootstrap-file" .}: InputFile
 
     staticPeers {.
       desc: "Connect to one or more trusted peers(ENR or enode URL)"
       defaultValue: @[]
       defaultValueDesc: ""
-      name: "static-peers" }: seq[string]
+      name: "static-peers" .}: seq[string]
 
     staticPeersFile {.
       desc: "Specifies a file of trusted peers addresses(ENR or enode URL). " &
             "Both line delimited or YAML format are supported"
       defaultValue: ""
-      name: "static-peers-file" }: InputFile
+      name: "static-peers-file" .}: InputFile
 
     reconnectMaxRetry* {.
       desc: "Specifies max number of retries if static peers disconnected/not connected. " &
             "0 = infinite."
       defaultValue: 0
-      name: "reconnect-max-retry" }: int
+      name: "reconnect-max-retry" .}: int
 
     reconnectInterval* {.
       desc: "Interval in seconds before next attempt to reconnect to static peers. Min 5 seconds."
       defaultValue: 15
-      name: "reconnect-interval" }: int
+      name: "reconnect-interval" .}: int
 
     listenAddress* {.
       desc: "Listening IP address for Ethereum P2P and Discovery traffic"
       defaultValue: defaultListenAddress
       defaultValueDesc: $defaultListenAddressDesc
-      name: "listen-address" }: IpAddress
+      name: "listen-address" .}: IpAddress
 
     tcpPort* {.
       desc: "Ethereum P2P network listening TCP port"
-      defaultValue: defaultPort
-      defaultValueDesc: $defaultPort
-      name: "tcp-port" }: Port
+      defaultValue: defaultExecutionPort
+      defaultValueDesc: $defaultExecutionPort
+      name: "tcp-port" .}: Port
 
-    udpPort* {.
+    udpPortFlag* {.
       desc: "Ethereum P2P network listening UDP port"
-      defaultValue: 0 # set udpPort defaultValue in `makeConfig`
       defaultValueDesc: "default to --tcp-port"
-      name: "udp-port" }: Port
+      name: "udp-port" .}: Option[Port]
 
     maxPeers* {.
       desc: "Maximum number of peers to connect to"
       defaultValue: 25
-      name: "max-peers" }: int
+      name: "max-peers" .}: int
 
     nat* {.
       desc: "Specify method to use for determining public address. " &
@@ -375,68 +374,68 @@ type
         " by stateless clients such as generation and storage of block witnesses" &
         " and serving these witnesses to peers over the p2p network."
       defaultValue: false
-      name: "stateless-provider" }: bool
+      name: "stateless-provider" .}: bool
 
     statelessWitnessValidation* {.
       hidden
       desc: "Enable full validation of execution witnesses."
       defaultValue: false
-      name: "stateless-witness-validation" }: bool
+      name: "stateless-witness-validation" .}: bool
 
     case cmd* {.
       command
-      defaultValue: NimbusCmd.noCommand }: NimbusCmd
+      defaultValue: NimbusCmd.executionClient .}: NimbusCmd
 
-    of noCommand:
+    of NimbusCmd.executionClient:
       httpPort* {.
         separator: "\pLOCAL SERVICES OPTIONS:"
         desc: "Listening port of the HTTP server(rpc, ws)"
         defaultValue: defaultHttpPort
         defaultValueDesc: $defaultHttpPort
-        name: "http-port" }: Port
+        name: "http-port" .}: Port
 
       httpAddress* {.
         desc: "Listening IP address of the HTTP server(rpc, ws)"
         defaultValue: defaultAdminListenAddress
         defaultValueDesc: $defaultAdminListenAddressDesc
-        name: "http-address" }: IpAddress
+        name: "http-address" .}: IpAddress
 
       rpcEnabled* {.
         desc: "Enable the JSON-RPC server"
         defaultValue: false
-        name: "rpc" }: bool
+        name: "rpc" .}: bool
 
       rpcApi {.
         desc: "Enable specific set of RPC API (available: eth, debug, admin)"
         defaultValue: @[]
         defaultValueDesc: $RpcFlag.Eth
-        name: "rpc-api" }: seq[string]
+        name: "rpc-api" .}: seq[string]
 
       wsEnabled* {.
         desc: "Enable the Websocket JSON-RPC server"
         defaultValue: false
-        name: "ws" }: bool
+        name: "ws" .}: bool
 
       wsApi {.
         desc: "Enable specific set of Websocket RPC API (available: eth, debug, admin)"
         defaultValue: @[]
         defaultValueDesc: $RpcFlag.Eth
-        name: "ws-api" }: seq[string]
+        name: "ws-api" .}: seq[string]
 
       historyExpiry* {.
         desc: "Enable the data from Portal Network"
         defaultValue: false
-        name: "history-expiry" }: bool
+        name: "history-expiry" .}: bool
 
       historyExpiryLimit* {.
         hidden
         desc: "Limit the number of blocks to be kept in history"
-        name: "debug-history-expiry-limit" }: Option[BlockNumber]
+        name: "debug-history-expiry-limit" .}: Option[BlockNumber]
 
       portalUrl* {.
         desc: "URL of the Portal JSON-RPC API"
         defaultValue: ""
-        name: "portal-url" }: string
+        name: "portal-url" .}: string
 
       engineApiEnabled* {.
         desc: "Enable the Engine API"
@@ -474,6 +473,12 @@ type
         defaultValueDesc: "\"jwt.hex\" in the data directory (see --data-dir)"
         name: "jwt-secret" .}: Option[InputFile]
 
+      jwtSecretValue* {.
+        hidden
+        desc: "Hex string with jwt secret"
+        defaultValueDesc: "\"jwt.hex\" in the data directory (see --data-dir)"
+        name: "debug-jwt-secret-value" .}: Option[string]
+
       beaconSyncTarget* {.
         hidden
         desc: "Manually set the initial sync target specified by its 32 byte" &
@@ -545,7 +550,7 @@ type
       blocksFile* {.
         argument
         desc: "One or more RLP encoded block(s) files"
-        name: "blocks-file" }: seq[InputFile]
+        name: "blocks-file" .}: seq[InputFile]
 
 func parseHexOrDec256(p: string): UInt256 {.raises: [ValueError].} =
   if startsWith(p, "0x"):
@@ -763,6 +768,9 @@ proc era1Dir*(config: ExecutionClientConf): string =
 proc eraDir*(config: ExecutionClientConf): string =
   string config.eraDirFlag.get(OutDir config.dataDir / "era")
 
+func udpPort*(config: ExecutionClientConf): Port =
+  config.udpPortFlag.get(config.tcpPort)
+
 func dbOptions*(config: ExecutionClientConf, noKeyCache = false): DbOptions =
   DbOptions.init(
     maxOpenFiles = config.rocksdbMaxOpenFiles,
@@ -796,11 +804,6 @@ proc makeConfig*(cmdLine = commandLineParams(), ignoreUnknown = false): Executio
 
   processNetworkParamsAndNetworkId(result)
 
-  if result.cmd == noCommand:
-    if result.udpPort == Port(0):
-      # if udpPort not set in cli, then
-      result.udpPort = result.tcpPort
-
 when isMainModule:
   # for testing purpose
   discard makeConfig()