feat(api): Integrate new VM into API server (no tracers)

core/bin/zksync_server/src/node_builder.rs

@@ -308,10 +308,12 @@ impl MainNodeBuilder {
             initial_writes_cache_size: rpc_config.initial_writes_cache_size() as u64,
             latest_values_cache_size: rpc_config.latest_values_cache_size() as u64,
         };
+        let vm_config = try_load_config!(self.configs.experimental_vm_config);
 
         // On main node we always use master pool sink.
         self.node.add_layer(MasterPoolSinkLayer);
-        self.node.add_layer(TxSenderLayer::new(
+
+        let layer = TxSenderLayer::new(
             TxSenderConfig::new(
                 &sk_config,
                 &rpc_config,
@@ -322,7 +324,10 @@ impl MainNodeBuilder {
             ),
             postgres_storage_caches_config,
             rpc_config.vm_concurrency_limit(),
-        ));
+        );
+        let layer =
+            layer.with_gas_estimation_vm_mode(vm_config.api_fast_vm_mode_for_gas_estimation);
+        self.node.add_layer(layer);
         Ok(self)
     }
 

core/lib/config/src/configs/experimental.rs

@@ -106,4 +106,8 @@ pub struct ExperimentalVmConfig {
     /// the new VM doesn't produce call traces and can diverge from the old VM!
     #[serde(default)]
     pub state_keeper_fast_vm_mode: FastVmMode,
+
+    /// Fast VM mode to use for gas estimation in the API server.
+    #[serde(default)]
+    pub api_fast_vm_mode_for_gas_estimation: FastVmMode,
 }

core/lib/config/src/testonly.rs

@@ -326,6 +326,7 @@ impl Distribution<configs::ExperimentalVmConfig> for EncodeDist {
         configs::ExperimentalVmConfig {
             playground: self.sample(rng),
             state_keeper_fast_vm_mode: gen_fast_vm_mode(rng),
+            api_fast_vm_mode_for_gas_estimation: gen_fast_vm_mode(rng),
         }
     }
 }

core/lib/env_config/src/vm_runner.rs

@@ -55,6 +55,7 @@ mod tests {
         let mut lock = MUTEX.lock();
         let config = r#"
             EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=new
+            EXPERIMENTAL_VM_API_FAST_VM_MODE_FOR_GAS_ESTIMATION=shadow
             EXPERIMENTAL_VM_PLAYGROUND_FAST_VM_MODE=shadow
             EXPERIMENTAL_VM_PLAYGROUND_DB_PATH=/db/vm_playground
             EXPERIMENTAL_VM_PLAYGROUND_FIRST_PROCESSED_BATCH=123
@@ -64,6 +65,10 @@ mod tests {
 
         let config = ExperimentalVmConfig::from_env().unwrap();
         assert_eq!(config.state_keeper_fast_vm_mode, FastVmMode::New);
+        assert_eq!(
+            config.api_fast_vm_mode_for_gas_estimation,
+            FastVmMode::Shadow
+        );
         assert_eq!(config.playground.fast_vm_mode, FastVmMode::Shadow);
         assert_eq!(config.playground.db_path.unwrap(), "/db/vm_playground");
         assert_eq!(config.playground.first_processed_batch, L1BatchNumber(123));

core/lib/multivm/src/lib.rs

@@ -16,7 +16,7 @@ pub use crate::{
         vm_1_3_2, vm_1_4_1, vm_1_4_2, vm_boojum_integration, vm_fast, vm_latest, vm_m5, vm_m6,
         vm_refunds_enhancement, vm_virtual_blocks,
     },
-    vm_instance::{FastVmInstance, LegacyVmInstance},
+    vm_instance::{is_supported_by_fast_vm, FastVmInstance, LegacyVmInstance},
 };
 
 mod glue;

core/lib/multivm/src/versions/vm_fast/tests/l1_tx_execution.rs

@@ -1,16 +1,21 @@
+use assert_matches::assert_matches;
 use ethabi::Token;
 use zksync_contracts::l1_messenger_contract;
 use zksync_system_constants::{BOOTLOADER_ADDRESS, L1_MESSENGER_ADDRESS};
 use zksync_types::{
     get_code_key, get_known_code_key,
     l2_to_l1_log::{L2ToL1Log, UserL2ToL1Log},
-    Execute, ExecuteTransactionCommon, U256,
+    Address, Execute, ExecuteTransactionCommon, U256,
 };
 use zksync_utils::{h256_to_u256, u256_to_h256};
 
 use crate::{
-    interface::{TxExecutionMode, VmExecutionMode, VmInterface, VmInterfaceExt},
+    interface::{
+        ExecutionResult, TxExecutionMode, VmExecutionMode, VmInterface, VmInterfaceExt,
+        VmRevertReason,
+    },
     utils::StorageWritesDeduplicator,
+    versions::testonly::ContractToDeploy,
     vm_fast::{
         tests::{
             tester::{TxType, VmTesterBuilder},
@@ -197,3 +202,42 @@ fn test_l1_tx_execution_high_gas_limit() {
 
     assert!(res.result.is_failed(), "The transaction should've failed");
 }
+
+#[test]
+fn test_l1_tx_execution_gas_estimation_with_low_gas() {
+    let counter_contract = read_test_contract();
+    let counter_address = Address::repeat_byte(0x11);
+    let mut vm = VmTesterBuilder::new()
+        .with_empty_in_memory_storage()
+        .with_base_system_smart_contracts(BASE_SYSTEM_CONTRACTS.clone())
+        .with_execution_mode(TxExecutionMode::EstimateFee)
+        .with_custom_contracts(vec![ContractToDeploy::new(
+            counter_contract,
+            counter_address,
+        )])
+        .with_random_rich_accounts(1)
+        .build();
+
+    let account = &mut vm.rich_accounts[0];
+    let mut tx = account.get_test_contract_transaction(
+        counter_address,
+        false,
+        None,
+        false,
+        TxType::L1 { serial_id: 0 },
+    );
+    let ExecuteTransactionCommon::L1(data) = &mut tx.common_data else {
+        unreachable!();
+    };
+    // This gas limit is chosen so that transaction starts getting executed by the bootloader, but then runs out of gas
+    // before its execution result is posted.
+    data.gas_limit = 15_000.into();
+
+    vm.vm.push_transaction(tx);
+    let res = vm.vm.execute(VmExecutionMode::OneTx);
+    assert_matches!(
+        &res.result,
+        ExecutionResult::Revert { output: VmRevertReason::General { msg, .. } }
+            if msg.contains("reverted with empty reason")
+    );
+}

core/lib/multivm/src/versions/vm_fast/vm.rs

@@ -49,8 +49,8 @@ use crate::{
     },
     vm_latest::{
         constants::{
-            get_vm_hook_params_start_position, get_vm_hook_position, OPERATOR_REFUNDS_OFFSET,
-            TX_GAS_LIMIT_OFFSET, VM_HOOK_PARAMS_COUNT,
+            get_result_success_first_slot, get_vm_hook_params_start_position, get_vm_hook_position,
+            OPERATOR_REFUNDS_OFFSET, TX_GAS_LIMIT_OFFSET, VM_HOOK_PARAMS_COUNT,
         },
         MultiVMSubversion,
     },
@@ -205,7 +205,22 @@ impl<S: ReadStorage, Tr: Tracer> Vm<S, Tr> {
                 }
                 Hook::TxHasEnded => {
                     if let VmExecutionMode::OneTx = execution_mode {
-                        break (last_tx_result.take().unwrap(), false);
+                        // The bootloader may invoke `TxHasEnded` hook without posting a tx result previously. One case when this can happen
+                        // is estimating gas for L1 transactions, if a transaction runs out of gas during execution.
+                        let tx_result = last_tx_result.take().unwrap_or_else(|| {
+                            let tx_has_failed = self.get_tx_result().is_zero();
+                            if tx_has_failed {
+                                let output = VmRevertReason::General {
+                                    msg: "Transaction reverted with empty reason. Possibly out of gas"
+                                        .to_string(),
+                                    data: vec![],
+                                };
+                                ExecutionResult::Revert { output }
+                            } else {
+                                ExecutionResult::Success { output: vec![] }
+                            }
+                        });
+                        break (tx_result, false);
                     }
                 }
                 Hook::AskOperatorForRefund => {
@@ -353,6 +368,12 @@ impl<S: ReadStorage, Tr: Tracer> Vm<S, Tr> {
             .unwrap()
     }
 
+    fn get_tx_result(&self) -> U256 {
+        let tx_idx = self.bootloader_state.current_tx();
+        let slot = get_result_success_first_slot(VM_VERSION) as usize + tx_idx;
+        self.read_word_from_bootloader_heap(slot)
+    }
+
     fn get_debug_log(&self) -> (String, String) {
         let hook_params = self.get_hook_params();
         let mut msg = u256_to_h256(hook_params[0]).as_bytes().to_vec();

core/lib/multivm/src/versions/vm_latest/tests/l1_tx_execution.rs

@@ -1,16 +1,20 @@
+use assert_matches::assert_matches;
 use ethabi::Token;
 use zksync_contracts::l1_messenger_contract;
 use zksync_system_constants::{BOOTLOADER_ADDRESS, L1_MESSENGER_ADDRESS};
 use zksync_test_account::Account;
 use zksync_types::{
     get_code_key, get_known_code_key,
     l2_to_l1_log::{L2ToL1Log, UserL2ToL1Log},
-    Execute, ExecuteTransactionCommon, K256PrivateKey, U256,
+    Address, Execute, ExecuteTransactionCommon, K256PrivateKey, U256,
 };
 use zksync_utils::u256_to_h256;
 
 use crate::{
-    interface::{TxExecutionMode, VmExecutionMode, VmInterface, VmInterfaceExt},
+    interface::{
+        ExecutionResult, TxExecutionMode, VmExecutionMode, VmInterface, VmInterfaceExt,
+        VmRevertReason,
+    },
     utils::StorageWritesDeduplicator,
     vm_latest::{
         tests::{
@@ -194,3 +198,39 @@ fn test_l1_tx_execution_high_gas_limit() {
 
     assert!(res.result.is_failed(), "The transaction should've failed");
 }
+
+#[test]
+fn test_l1_tx_execution_gas_estimation_with_low_gas() {
+    let counter_contract = read_test_contract();
+    let counter_address = Address::repeat_byte(0x11);
+    let mut vm = VmTesterBuilder::new(HistoryEnabled)
+        .with_empty_in_memory_storage()
+        .with_base_system_smart_contracts(BASE_SYSTEM_CONTRACTS.clone())
+        .with_execution_mode(TxExecutionMode::EstimateFee)
+        .with_custom_contracts(vec![(counter_contract, counter_address, false)])
+        .with_random_rich_accounts(1)
+        .build();
+
+    let account = &mut vm.rich_accounts[0];
+    let mut tx = account.get_test_contract_transaction(
+        counter_address,
+        false,
+        None,
+        false,
+        TxType::L1 { serial_id: 0 },
+    );
+    let ExecuteTransactionCommon::L1(data) = &mut tx.common_data else {
+        unreachable!();
+    };
+    // This gas limit is chosen so that transaction starts getting executed by the bootloader, but then runs out of gas
+    // before its execution result is posted.
+    data.gas_limit = 15_000.into();
+
+    vm.vm.push_transaction(tx);
+    let res = vm.vm.execute(VmExecutionMode::OneTx);
+    assert_matches!(
+        &res.result,
+        ExecutionResult::Revert { output: VmRevertReason::General { msg, .. } }
+            if msg.contains("reverted with empty reason")
+    );
+}

core/lib/multivm/src/vm_instance.rs

@@ -1,6 +1,6 @@
 use std::mem;
 
-use zksync_types::{vm::VmVersion, Transaction};
+use zksync_types::{vm::VmVersion, ProtocolVersionId, Transaction};
 use zksync_vm2::interface::Tracer;
 
 use crate::{
@@ -225,7 +225,7 @@ pub type ShadowedFastVm<S, Tr = ()> = ShadowVm<
 
 /// Fast VM variants.
 #[derive(Debug)]
-pub enum FastVmInstance<S: ReadStorage, Tr> {
+pub enum FastVmInstance<S: ReadStorage, Tr = ()> {
     /// Fast VM running in isolation.
     Fast(crate::vm_fast::Vm<ImmutableStorageView<S>, Tr>),
     /// Fast VM shadowed by the latest legacy VM.
@@ -328,3 +328,11 @@ impl<S: ReadStorage, Tr: Tracer + Default + 'static> FastVmInstance<S, Tr> {
         Self::Shadowed(ShadowedFastVm::new(l1_batch_env, system_env, storage_view))
     }
 }
+
+/// Checks whether the protocol version is supported by the fast VM.
+pub fn is_supported_by_fast_vm(protocol_version: ProtocolVersionId) -> bool {
+    matches!(
+        protocol_version.into(),
+        VmVersion::Vm1_5_0IncreasedBootloaderMemory
+    )
+}

core/lib/protobuf_config/src/experimental.rs

@@ -7,6 +7,14 @@ use zksync_protobuf::{repr::ProtoRepr, required};
 
 use crate::{proto::experimental as proto, read_optional_repr};
 
+fn parse_vm_mode(raw: Option<i32>) -> anyhow::Result<FastVmMode> {
+    Ok(raw
+        .map(proto::FastVmMode::try_from)
+        .transpose()
+        .context("fast_vm_mode")?
+        .map_or_else(FastVmMode::default, |mode| mode.parse()))
+}
+
 impl ProtoRepr for proto::Db {
     type Type = configs::ExperimentalDBConfig;
 
@@ -105,12 +113,10 @@ impl ProtoRepr for proto::Vm {
     fn read(&self) -> anyhow::Result<Self::Type> {
         Ok(Self::Type {
             playground: read_optional_repr(&self.playground).unwrap_or_default(),
-            state_keeper_fast_vm_mode: self
-                .state_keeper_fast_vm_mode
-                .map(proto::FastVmMode::try_from)
-                .transpose()
-                .context("fast_vm_mode")?
-                .map_or_else(FastVmMode::default, |mode| mode.parse()),
+            state_keeper_fast_vm_mode: parse_vm_mode(self.state_keeper_fast_vm_mode)?,
+            api_fast_vm_mode_for_gas_estimation: parse_vm_mode(
+                self.api_fast_vm_mode_for_gas_estimation,
+            )?,
         })
     }
 
@@ -120,6 +126,9 @@ impl ProtoRepr for proto::Vm {
             state_keeper_fast_vm_mode: Some(
                 proto::FastVmMode::new(this.state_keeper_fast_vm_mode).into(),
             ),
+            api_fast_vm_mode_for_gas_estimation: Some(
+                proto::FastVmMode::new(this.api_fast_vm_mode_for_gas_estimation).into(),
+            ),
         }
     }
 }

core/lib/protobuf_config/src/proto/config/experimental.proto

@@ -37,4 +37,5 @@ message VmPlayground {
 message Vm {
   optional VmPlayground playground = 1; // optional
   optional FastVmMode state_keeper_fast_vm_mode = 2; // optional; if not set, fast VM is not used
+  optional FastVmMode api_fast_vm_mode_for_gas_estimation = 3; // optional; if not set, fast VM is not used
 }

core/lib/vm_executor/Cargo.toml

@@ -23,3 +23,7 @@ tokio.workspace = true
 anyhow.workspace = true
 tracing.workspace = true
 vise.workspace = true
+
+[dev-dependencies]
+assert_matches.workspace = true
+test-casing.workspace = true

core/lib/vm_executor/src/oneshot/metrics.rs

@@ -50,7 +50,7 @@ pub(super) fn report_vm_memory_metrics(
     tx_id: &str,
     memory_metrics: &VmMemoryMetrics,
     vm_execution_took: Duration,
-    storage_metrics: &StorageViewStats,
+    storage_stats: &StorageViewStats,
 ) {
     MEMORY_METRICS.event_sink_size[&SizeType::Inner].observe(memory_metrics.event_sink_inner);
     MEMORY_METRICS.event_sink_size[&SizeType::History].observe(memory_metrics.event_sink_history);
@@ -65,10 +65,18 @@ pub(super) fn report_vm_memory_metrics(
 
     MEMORY_METRICS
         .storage_view_cache_size
-        .observe(storage_metrics.cache_size);
+        .observe(storage_stats.cache_size);
     MEMORY_METRICS
         .full
-        .observe(memory_metrics.full_size() + storage_metrics.cache_size);
+        .observe(memory_metrics.full_size() + storage_stats.cache_size);
 
-    STORAGE_METRICS.observe(&format!("Tx {tx_id}"), vm_execution_took, storage_metrics);
+    report_vm_storage_metrics(tx_id, vm_execution_took, storage_stats);
+}
+
+pub(super) fn report_vm_storage_metrics(
+    tx_id: &str,
+    vm_execution_took: Duration,
+    storage_stats: &StorageViewStats,
+) {
+    STORAGE_METRICS.observe(&format!("Tx {tx_id}"), vm_execution_took, storage_stats);
 }