Zksync era integration tests (#208)

* adapt to zksync-era tests

* Remove hash_map from Storage trait

* Fix increment_tx_number not cleaning transient_storage

* Fix decommit opcode

Fixes decommit opcode, which was not account for if the code had already been decommitted

* Various far_call fixes

Adds mandated gass, handles decommit cost err, and fixes the params of mimic call

* Add stipend param to frame and decrease it in panics and reverts

* Add no refund storage_read for far_call decommit_code_hash

* Remove rollbacks from main context

* Reimplement vec snapshots

* Add external snapshot(whole vm)

* Add storage cache and util functio to get storage changes from initial

* Refactor state pub fields

* Add new function to get storage changes from snapshot or initial

* Update zksync-era submodule

* Update era-compiler-tester submodule

* Address clippy warnings

* More clippy warnings

* Update zksync-era submodule

* ci: add submodule step for era

* Address review comments

* Add cache to storage_read

* Update zksync-era submodule

* Address review comments

* Add docs to state

* Fix storage_read

* Fix initial value read in storage_changes

* Update era-compiler-tester submodule

---------

Co-authored-by: Juan Munoz <[email protected]>
Co-authored-by: Fran <[email protected]>

.github/workflows/ci.yml

@@ -621,7 +621,7 @@ jobs:
         with:
           path: ${{ github.workspace }}/era_vm
 
-      - name: Setup compiler-tester submodule
+      - name: Setup era submodule
         working-directory: ${{ github.workspace }}/era_vm
         run: |
           make submodules

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "era-compiler-tester"]
-	path = era-compiler-tester
-	url = https://github.com/lambdaclass/era-compiler-tester.git
 [submodule "zksync-era"]
 	path = zksync-era
 	url = https://github.com/lambdaclass/zksync-era.git
+[submodule "era-compiler-tester"]
+	path = era-compiler-tester
+	url = https://github.com/lambdaclass/era-compiler-tester.git

src/call_frame.rs

@@ -11,6 +11,7 @@ pub struct CallFrame {
     pub exception_handler: u64,
     pub sp: u32,
     pub snapshot: StateSnapshot,
+    pub stipend: Saturating<u32>,
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -54,7 +55,7 @@ impl Context {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         program_code: Vec<U256>,
-        gas_stipend: u32,
+        gas: u32,
         contract_address: Address,
         code_address: Address,
         caller: Address,
@@ -65,9 +66,10 @@ impl Context {
         context_u128: u128,
         snapshot: StateSnapshot,
         is_static: bool,
+        stipend: u32,
     ) -> Self {
         Self {
-            frame: CallFrame::new_far_call_frame(gas_stipend, exception_handler, snapshot),
+            frame: CallFrame::new_far_call_frame(gas, stipend, exception_handler, snapshot),
             near_call_frames: vec![],
             contract_address,
             caller,
@@ -89,13 +91,15 @@ impl Context {
 
 impl CallFrame {
     pub fn new_far_call_frame(
-        gas_stipend: u32,
+        gas: u32,
+        stipend: u32,
         exception_handler: u64,
         snapshot: StateSnapshot,
     ) -> Self {
         Self {
             pc: 0,
-            gas_left: Saturating(gas_stipend),
+            stipend: Saturating(stipend),
+            gas_left: Saturating(gas),
             exception_handler,
             sp: 0,
             snapshot,
@@ -106,14 +110,15 @@ impl CallFrame {
     pub fn new_near_call_frame(
         sp: u32,
         pc: u64,
-        gas_stipend: u32,
+        gas: u32,
         exception_handler: u64,
         snapshot: StateSnapshot,
     ) -> Self {
         Self {
             pc,
-            gas_left: Saturating(gas_stipend),
+            gas_left: Saturating(gas),
             exception_handler,
+            stipend: Saturating(0),
             sp,
             snapshot,
         }

src/execution.rs

@@ -29,6 +29,7 @@ pub struct Heap {
 }
 
 #[derive(Debug, Clone, PartialEq)]
+// represents the vm execution state
 pub struct Execution {
     // The first register, r0, is actually always zero and not really used.
     // Writing to it does nothing.
@@ -50,6 +51,23 @@ pub struct Execution {
     pub use_hooks: bool,
 }
 
+#[derive(Debug, Clone, PartialEq)]
+// a saved state of the vm execution
+pub struct ExecutionSnapshot {
+    pub registers: [TaggedValue; 15],
+    pub flag_lt_of: bool,
+    pub flag_gt: bool,
+    pub flag_eq: bool,
+    pub running_contexts: Vec<Context>,
+    pub tx_number: u64,
+    pub heaps: Heaps,
+    pub register_context_u128: u128,
+    pub default_aa_code_hash: [u8; 32],
+    pub evm_interpreter_code_hash: [u8; 32],
+    pub hook_address: u32,
+    pub use_hooks: bool,
+}
+
 impl Execution {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -87,6 +105,7 @@ impl Execution {
             context_u128,
             StateSnapshot::default(),
             false,
+            0,
         );
 
         let heaps = Heaps::new(calldata);
@@ -142,6 +161,7 @@ impl Execution {
         context_u128: u128,
         snapshot: StateSnapshot,
         is_static: bool,
+        stipend: u32,
     ) -> Result<(), EraVmError> {
         let new_context = Context::new(
             program_code,
@@ -156,6 +176,7 @@ impl Execution {
             context_u128,
             snapshot,
             is_static,
+            stipend,
         );
         self.running_contexts.push(new_context);
         Ok(())
@@ -299,7 +320,7 @@ impl Execution {
     }
 
     pub fn increase_gas(&mut self, to_add: u32) -> Result<(), EraVmError> {
-        self.current_frame_mut()?.gas_left += to_add;
+        self.current_frame_mut()?.gas_left += Saturating(to_add);
         Ok(())
     }
 
@@ -319,6 +340,37 @@ impl Execution {
     pub fn in_far_call(&self) -> bool {
         self.running_contexts.len() > 1
     }
+
+    pub fn snapshot(&self) -> ExecutionSnapshot {
+        ExecutionSnapshot {
+            default_aa_code_hash: self.default_aa_code_hash,
+            evm_interpreter_code_hash: self.evm_interpreter_code_hash,
+            flag_eq: self.flag_eq,
+            flag_gt: self.flag_gt,
+            flag_lt_of: self.flag_lt_of,
+            heaps: self.heaps.clone(),
+            hook_address: self.hook_address,
+            register_context_u128: self.register_context_u128,
+            registers: self.registers,
+            running_contexts: self.running_contexts.clone(),
+            tx_number: self.tx_number,
+            use_hooks: self.use_hooks,
+        }
+    }
+    pub fn rollback(&mut self, snapshot: ExecutionSnapshot) {
+        self.default_aa_code_hash = snapshot.default_aa_code_hash;
+        self.evm_interpreter_code_hash = snapshot.evm_interpreter_code_hash;
+        self.flag_eq = snapshot.flag_eq;
+        self.flag_gt = snapshot.flag_gt;
+        self.flag_lt_of = snapshot.flag_lt_of;
+        self.heaps = snapshot.heaps;
+        self.hook_address = snapshot.hook_address;
+        self.register_context_u128 = snapshot.register_context_u128;
+        self.registers = snapshot.registers;
+        self.running_contexts = snapshot.running_contexts;
+        self.tx_number = snapshot.tx_number;
+        self.use_hooks = snapshot.use_hooks;
+    }
 }
 
 impl Default for Stack {

src/op_handlers/context.rs

@@ -81,7 +81,12 @@ pub fn set_context_u128(vm: &mut Execution, opcode: &Opcode) -> Result<(), EraVm
     Ok(())
 }
 
-pub fn increment_tx_number(vm: &mut Execution, _opcode: &Opcode) -> Result<(), EraVmError> {
+pub fn increment_tx_number(
+    vm: &mut Execution,
+    _opcode: &Opcode,
+    state: &mut VMState,
+) -> Result<(), EraVmError> {
     vm.tx_number += 1;
+    state.clear_transient_storage();
     Ok(())
 }

src/op_handlers/far_call.rs

@@ -1,8 +1,11 @@
 use u256::{H160, U256};
 use zkevm_opcode_defs::{
     ethereum_types::Address,
-    system_params::{DEPLOYER_SYSTEM_CONTRACT_ADDRESS_LOW, EVM_SIMULATOR_STIPEND},
-    FarCallOpcode,
+    system_params::{
+        DEPLOYER_SYSTEM_CONTRACT_ADDRESS_LOW, EVM_SIMULATOR_STIPEND,
+        MSG_VALUE_SIMULATOR_ADDITIVE_COST,
+    },
+    FarCallOpcode, ADDRESS_MSG_VALUE,
 };
 
 use crate::{
@@ -102,6 +105,7 @@ fn far_call_params_from_register(
 
     let maximum_gas =
         gas_left / FAR_CALL_GAS_SCALAR_MODIFIER_DIVISOR * FAR_CALL_GAS_SCALAR_MODIFIER_DIVIDEND;
+
     ergs_passed = ergs_passed.min(maximum_gas);
 
     source.to_little_endian(&mut args);
@@ -136,7 +140,7 @@ fn decommit_code_hash(
     let storage_key = StorageKey::new(deployer_system_contract_address, address_into_u256(address));
 
     // reading when decommiting doesn't refund
-    let (code_info, _) = state.storage_read(storage_key);
+    let code_info = state.storage_read_with_no_refund(storage_key);
     let mut code_info_bytes = [0; 32];
     code_info.to_big_endian(&mut code_info_bytes);
 
@@ -230,8 +234,10 @@ pub fn far_call(
     )?;
 
     // Unlike all other gas costs, this one is not paid if low on gas.
-    if decommit_cost < vm.gas_left()? {
+    if decommit_cost <= vm.gas_left()? {
         vm.decrease_gas(decommit_cost)?;
+    } else {
+        return Err(EraVmError::DecommitFailed);
     }
 
     let FarCallParams {
@@ -240,16 +246,26 @@ pub fn far_call(
         ..
     } = far_call_params_from_register(src0, vm)?;
 
+    let mandated_gas = if abi.is_system_call && src1.value == ADDRESS_MSG_VALUE.into() {
+        MSG_VALUE_SIMULATOR_ADDITIVE_COST
+    } else {
+        0
+    };
+
+    // mandated gas can surprass the 63/64 limit
+    let ergs_passed = ergs_passed + mandated_gas;
+
     vm.decrease_gas(ergs_passed)?;
 
     let stipend = if is_evm { EVM_SIMULATOR_STIPEND } else { 0 };
 
-    let ergs_passed = ergs_passed
+    let ergs_passed = (ergs_passed)
         .checked_add(stipend)
         .expect("stipend must not cause overflow");
 
     let program_code = state
         .decommit(code_key)
+        .0
         .ok_or(StorageError::KeyNotPresent)?;
     let new_heap = vm.heaps.allocate();
     let new_aux_heap = vm.heaps.allocate();
@@ -270,31 +286,26 @@ pub fn far_call(
                 vm.register_context_u128,
                 snapshot,
                 is_new_frame_static && !is_evm,
+                stipend,
             )?;
         }
         FarCallOpcode::Mimic => {
-            let mut caller_bytes = [0; 32];
-            let caller = vm.get_register(15).value;
-            caller.to_big_endian(&mut caller_bytes);
-
-            let mut caller_bytes_20: [u8; 20] = [0; 20];
-            for (i, byte) in caller_bytes[12..].iter().enumerate() {
-                caller_bytes_20[i] = *byte;
-            }
+            let caller = address_from_u256(&vm.get_register(15).value);
 
             vm.push_far_call_frame(
                 program_code,
                 ergs_passed,
                 contract_address,
                 contract_address,
-                H160::from(caller_bytes_20),
+                caller,
                 new_heap,
                 new_aux_heap,
                 forward_memory.page,
                 exception_handler,
                 vm.register_context_u128,
                 snapshot,
                 is_new_frame_static && !is_evm,
+                stipend,
             )?;
         }
         FarCallOpcode::Delegate => {
@@ -314,6 +325,7 @@ pub fn far_call(
                 this_context.context_u128,
                 snapshot,
                 is_new_frame_static && !is_evm,
+                stipend,
             )?;
         }
     };

src/op_handlers/opcode_decommit.rs

@@ -1,3 +1,5 @@
+use zkevm_opcode_defs::{BlobSha256Format, ContractCodeSha256Format, VersionedHashLen32};
+
 use crate::{
     address_operands::{address_operands_read, address_operands_store},
     eravm_error::{EraVmError, HeapError},
@@ -18,11 +20,25 @@ pub fn opcode_decommit(
 
     let preimage_len_in_bytes = zkevm_opcode_defs::system_params::NEW_KERNEL_FRAME_MEMORY_STIPEND;
 
-    vm.decrease_gas(extra_cost)?;
+    let mut buffer = [0u8; 32];
+    code_hash.to_big_endian(&mut buffer);
+
+    // gas is payed in advance
+    if vm.decrease_gas(extra_cost).is_err()
+        || (!ContractCodeSha256Format::is_valid(&buffer) && !BlobSha256Format::is_valid(&buffer))
+    {
+        // we don't actually return an err here
+        vm.set_register(1, TaggedValue::zero());
+        return Ok(());
+    }
+
+    let (code, was_decommited) = state.decommit(code_hash);
+    if was_decommited {
+        // refund it
+        vm.increase_gas(extra_cost)?;
+    };
 
-    let code = state
-        .decommit(code_hash)
-        .ok_or(EraVmError::DecommitFailed)?;
+    let code = code.ok_or(EraVmError::DecommitFailed)?;
 
     let code_len_in_bytes = code.len() * 32;
     let id = vm.heaps.allocate();

src/op_handlers/ret.rs

@@ -63,7 +63,7 @@ pub fn ret(
         vm.clear_registers();
         vm.set_register(1, result);
         let previous_frame = vm.pop_frame()?;
-        vm.current_frame_mut()?.gas_left += previous_frame.gas_left;
+        vm.increase_gas((previous_frame.gas_left - previous_frame.stipend).0)?;
         if is_failure {
             state.rollback(previous_frame.snapshot);
             vm.current_frame_mut()?.pc = previous_frame.exception_handler;
@@ -73,9 +73,10 @@ pub fn ret(
 
         Ok(false)
     } else {
-        if is_failure {
-            state.rollback(vm.current_frame()?.snapshot.clone());
-        }
+        // The initial frame is not rolled back, even if it fails.
+        // It is the caller's job to clean up when the execution as a whole fails because
+        // the caller may take external snapshots while the VM is in the initial frame and
+        // these would break were the initial frame to be rolled back.
         if return_type == RetOpcode::Panic {
             return Ok(true);
         }
@@ -94,6 +95,7 @@ pub fn inexplicit_panic(vm: &mut Execution, state: &mut VMState) -> Result<bool,
         let previous_frame = vm.pop_frame()?;
         vm.current_frame_mut()?.pc = previous_frame.exception_handler;
         vm.current_frame_mut()?.gas_left += previous_frame.gas_left;
+
         state.rollback(previous_frame.snapshot);
 
         Ok(false)
@@ -103,12 +105,11 @@ pub fn inexplicit_panic(vm: &mut Execution, state: &mut VMState) -> Result<bool,
         vm.clear_registers();
         vm.set_register(1, result);
         let previous_frame = vm.pop_frame()?;
-        vm.current_frame_mut()?.gas_left += previous_frame.gas_left;
+        vm.increase_gas((previous_frame.gas_left - previous_frame.stipend).0)?;
         vm.current_frame_mut()?.pc = previous_frame.exception_handler;
         state.rollback(previous_frame.snapshot);
         Ok(false)
     } else {
-        state.rollback(vm.current_frame()?.snapshot.clone());
         Ok(true)
     }
 }