Merge pull request #22 from yungwine/liteclient

Liteclient updates

.github/workflows/python-package.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest==7.4.2 pytest-asyncio
+        python -m pip install flake8 pytest==7.4.2 pytest-asyncio pytvm
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |

pytoniq/liteclient/_balancer_codegen.py

@@ -23,7 +23,7 @@ def main():
                 continue
             if 'async def' in line:
                 name = line[line.index('    async def ') + 14: line.index('(')]
-                if name in exceptions:
+                if name in exceptions or name.startswith('_'):
                     continue
                 tmp += line
             if tmp and ':\n' in line:

pytoniq/liteclient/balancer.py

@@ -326,6 +326,17 @@ async def run_get_method(self, address: typing.Union[Address, str],
                              , **kwargs) -> list:
         return await self.execute_method('run_get_method', **self._get_args(locals())) 
 
+    async def run_get_method_remote(self, address: typing.Union[Address, str],
+                                    method: typing.Union[int, str], stack: list,
+                                    block: BlockIdExt = None
+                                    , **kwargs) -> list:
+        return await self.execute_method('run_get_method_remote', **self._get_args(locals())) 
+
+    async def run_get_method_local(self, address: typing.Union[Address, str],
+                                   method: typing.Union[int, str], stack: list,
+                                   block: BlockIdExt = None, gas_limit: int = 300000, **kwargs) -> list:
+        return await self.execute_method('run_get_method_local', **self._get_args(locals())) 
+
     async def raw_get_shard_info(self, block: typing.Optional[BlockIdExt] = None,
                                  wc: int = 0, shard: int = -9223372036854775808,
                                  exact: bool = True
@@ -388,6 +399,18 @@ async def get_config_params(self, params: typing.List[int], blk: typing.Optional
     async def get_libraries(self, library_list: typing.List[typing.Union[bytes, str]], **kwargs) -> typing.Dict[str, typing.Optional[Cell]]:
         return await self.execute_method('get_libraries', **self._get_args(locals())) 
 
+    async def get_out_msg_queue_sizes(self, wc: int = None, shard: int = None, **kwargs):
+        return await self.execute_method('get_out_msg_queue_sizes', **self._get_args(locals())) 
+
+    async def nonfinal_get_validator_groups(self, wc: int = None, shard: int = None, **kwargs):
+        return await self.execute_method('nonfinal_get_validator_groups', **self._get_args(locals())) 
+
+    async def nonfinal_raw_get_candidate(self, candidate_id: dict, **kwargs):
+        return await self.execute_method('nonfinal_raw_get_candidate', **self._get_args(locals())) 
+
+    async def nonfinal_get_candidate(self, candidate_id: dict, **kwargs):
+        return await self.execute_method('nonfinal_get_candidate', **self._get_args(locals())) 
+
     async def get_shard_block_proof(self, blk: BlockIdExt, prove_mc: bool = False, **kwargs):
         return await self.execute_method('get_shard_block_proof', **self._get_args(locals())) 
 

pytoniq/liteclient/client.py

@@ -4,9 +4,11 @@
 import asyncio
 import socket
 import struct
+import time
 import typing
 
 import requests
+from pytoniq_core import HashMap, Builder
 
 from .sync import choose_key_block, sync
 from .utils import init_mainnet_block, init_testnet_block
@@ -25,7 +27,7 @@
 from pytoniq_core.tlb.utils import deserialize_shard_hashes
 
 from pytoniq_core.tlb.vm_stack import VmStack
-from pytoniq_core.tlb.block import Block, ShardDescr, BinTree, ShardStateUnsplit, KeyExtBlkRef
+from pytoniq_core.tlb.block import Block, ShardDescr, BinTree, ShardStateUnsplit, KeyExtBlkRef, BlockExtra
 from pytoniq_core.tlb.account import Account, SimpleAccount, ShardAccount, AccountBlock
 
 
@@ -105,6 +107,11 @@ def __init__(self,
         self.adnl_query_sch = self.schemas.get_by_name('adnl.message.query')
         self.ls_query_sch = self.schemas.get_by_name('liteServer.query')
 
+        """########### Get methods ###########"""
+        self._block_states = {}  # block root hash : block state
+        self.libs = {}  # library hash : library cell
+        self.configs = {}  # config hash : config cell
+
     def encrypt(self, data: bytes) -> bytes:
         return aes_ctr_encrypt(self.enc_sipher, data)
 
@@ -446,6 +453,8 @@ async def raw_get_account_state(self, address: typing.Union[str, Address],
             check_shard_proof(shard_proof=result['shard_proof'], blk=block, shrd_blk=shrd_blk)
             if not trusted and not self.trust_level:
                 await self.get_mc_block_proof(known_block=self.last_key_block, target_block=block)
+        if address.hash_part in self._block_states:
+            self._block_states[address.hash_part] = (result['proof'], result['shard_proof'])
         shard_account = check_account_proof(proof=result['proof'], shrd_blk=shrd_blk, address=address, account_state_root=account_state_root, return_account_descr=True)
         account = Account.deserialize(account_state_root.begin_parse())
         full_shard_account_cell = begin_cell().store_bytes(shard_account.cell.begin_parse().load_bytes(40)).store_ref(account_state_root).end_cell()
@@ -465,6 +474,14 @@ async def run_get_method(self, address: typing.Union[Address, str],
                              method: typing.Union[int, str], stack: list,
                              block: BlockIdExt = None
                              ) -> list:
+        return await self.run_get_method_remote(address, method, stack, block)  # will be replaced with run_get_method_local in future
+
+    async def run_get_method_remote(self, address: typing.Union[Address, str],
+                                    method: typing.Union[int, str], stack: list,
+                                    block: BlockIdExt = None
+                                    ) -> list:
+        if self.trust_level <= 1:
+            self.logger.warning('remote get method result is not provable, use run_get_method_local for local tvm execution')
         mode = 7  # 111
         if block is None:
             block = self.last_mc_block
@@ -496,6 +513,92 @@ async def run_get_method(self, address: typing.Union[Address, str],
 
         return VmStack.deserialize(Slice.one_from_boc(result['result']))
 
+    async def _get_config_cell(self, blk: BlockIdExt):
+        res = await self.liteserver_request('getConfigAll', {'mode': 0, 'id': blk.to_dict()})
+        config_proof = Cell.one_from_boc(res['config_proof'])
+        shard = ShardStateUnsplit.deserialize(config_proof[0].begin_parse())
+        config = shard.custom.config.config
+        hm = HashMap(32, value_serializer=lambda src, dest: dest.store_ref(src.to_cell()))
+        hm.map = config
+        return hm.serialize()
+
+    @staticmethod
+    def _find_libs(cell: Cell, libs: list):
+        if cell.type_ == 2:
+            libs.append(cell.begin_parse().preload_bytes(32))
+            return True
+        res = False
+        for ref in cell.refs:  # trick to avoid copying, don't repeat this at home
+            if LiteClient._find_libs(ref, libs):
+                res = True
+        return res
+
+    async def run_get_method_local(self, address: typing.Union[Address, str],
+                                   method: typing.Union[int, str], stack: list,
+                                   block: BlockIdExt = None, gas_limit: int = 300000) -> list:
+        if block is None:
+            block = self.last_mc_block
+        try:
+            from pytvm.tvm_emulator import TvmEmulator
+        except ImportError:
+            raise ImportError('pytvm is required to run get method locally. Use `pip install "pytoniq[tvm]"` or `pip install pytvm`')
+        if isinstance(address, Address):
+            address = address.to_str()
+        hash_part = Address(address).hash_part
+        self._block_states[hash_part] = None
+        try:
+            _, account = await self.raw_get_account_state(address, block)
+        finally:
+            shard_state_boc, mc_state_boc = self._block_states.pop(hash_part)
+        state = account.account.storage.state
+        if state.type_ != 'account_active':
+            raise RunGetMethodError(address=address, method=method, exit_code=-256)
+        emulator = TvmEmulator(
+            code=account.account.storage.state.state_init.code,
+            data=account.account.storage.state.state_init.data
+        )
+        emulator.set_gas_limit(gas_limit)
+        # set c7
+
+        # get config
+        config = Cell.from_boc(mc_state_boc)[1][0][3][1]
+        assert config.type_ == 1  # pruned branch
+        config_hash = config.get_hash(0)
+        if config_hash not in self.configs:
+            self.configs[config_hash] = await self._get_config_cell(block)
+        sstate = ShardStateUnsplit.deserialize(Cell.from_boc(shard_state_boc)[1][0].begin_parse())
+        emulator.set_c7(
+            address=address,
+            unixtime=sstate.gen_utime,
+            balance=account.account.storage.balance.grams,
+            rand_seed_hex=get_random(32).hex(),
+            config=self.configs[config_hash]
+        )
+
+        # set libs
+        libs = []
+        result_libs = {}
+        self._find_libs(account.account.storage.state.state_init.code, libs)
+        libs = [i for i in libs if i.hex() not in self.libs]
+        libs = [libs[i:i + 16] for i in range(0, len(libs), 16)]  # split libs into 16-element chunks
+        for i in libs:
+            result_libs |= await self.get_libraries(i)
+
+        def value_serializer(dest: Builder, src: Cell):
+            if src is not None:
+                dest.store_uint(0, 2).store_ref(src).store_maybe_ref(None)
+        hm = HashMap(256, value_serializer=value_serializer)
+        hm.map = self.libs
+        if self.libs:
+            emulator.set_libraries(hm.serialize())
+
+        # todo set prev blocks info
+
+        result = emulator.run_get_method(method, stack)
+        if result['vm_exit_code'] != 0:
+            raise RunGetMethodError(address=address, method=method, exit_code=result['vm_exit_code'])
+        return result['stack']
+
     async def raw_get_shard_info(self, block: typing.Optional[BlockIdExt] = None,
                                  wc: int = 0, shard: int = -9223372036854775808,
                                  exact: bool = True
@@ -542,17 +645,18 @@ async def raw_get_all_shards_info(self, block: typing.Optional[BlockIdExt] = Non
                 await self.get_mc_block_proof(known_block=self.last_key_block, target_block=block)
 
             proof_cells = Cell.from_boc(result['proof'])
+            if len(proof_cells) == 2:
+                state_hash = check_block_header_proof(proof_cells[0][0], block_hash=block.root_hash, store_state_hash=True)
+                check_proof(proof_cells[1], state_hash)
 
-            state_hash = check_block_header_proof(proof_cells[0][0], block_hash=block.root_hash, store_state_hash=True)
-
-            check_proof(proof_cells[1], state_hash)
-
-            shard_state = ShardStateUnsplit.deserialize(proof_cells[1][0].begin_parse())
+                shard_state = ShardStateUnsplit.deserialize(proof_cells[1][0].begin_parse())
 
-            assert shard_state.shard_id.workchain_id == block.workchain
-            assert shard_state.seq_no == block.seqno
-
-            assert shard_hashes_cell[0].get_hash(0) == proof_cells[1][0][3][0].get_hash(0)  # masterchain_state_extra -> shard_hashes
+                assert shard_state.shard_id.workchain_id == block.workchain
+                assert shard_state.seq_no == block.seqno
+                assert shard_hashes_cell[0].get_hash(0) == proof_cells[1][0][3][0].get_hash(0)  # masterchain_state_extra -> shard_hashes
+            else:
+                check_block_header_proof(proof_cells[0][0], block_hash=block.root_hash, store_state_hash=False)
+                assert shard_hashes_cell[0].get_hash(0) == proof_cells[0][0][3][3][0].get_hash(0)  # masterchain_state_extra -> shard_hashes
 
         return deserialize_shard_hashes(shard_hashes_cell.begin_parse())
 
@@ -950,6 +1054,37 @@ async def get_libraries(self, library_list: typing.List[typing.Union[bytes, str]
 
         return result
 
+    async def get_out_msg_queue_sizes(self, wc: int = None, shard: int = None):
+        """
+        If wc and shard are not None, returns queue size for all children shards of the provided shard.
+        If both are None, returns queue size for all shards for all workchains.
+        """
+        data = {}
+        mode = 0b0
+        assert not (wc is None) ^ (shard is None), 'workchain and shard must be both set or both not set'
+        if wc is not None or shard is not None:
+            data['wc'] = wc
+            data['shard'] = shard
+            mode += 0b01
+        return await self.liteserver_request('getOutMsgQueueSizes', data | {'mode': mode})
+
+    async def nonfinal_get_validator_groups(self, wc: int = None, shard: int = None):
+        data = {}
+        mode = 0b0
+        assert not (wc is None) ^ (shard is None), 'workchain and shard must be both set or both not set'
+        if wc is not None or shard is not None:
+            data['wc'] = wc
+            data['shard'] = shard
+            mode = 3
+        return await self.liteserver_request('nonfinal.getValidatorGroups', data | {'mode': mode})
+
+    async def nonfinal_raw_get_candidate(self, candidate_id: dict):
+        return await self.liteserver_request('nonfinal.getCandidate', {'id': candidate_id})
+
+    async def nonfinal_get_candidate(self, candidate_id: dict):
+        resp = await self.nonfinal_raw_get_candidate(candidate_id)
+        return Block.deserialize(Slice.one_from_boc(resp['data']))
+
     async def get_shard_block_proof(self, blk: BlockIdExt, prove_mc: bool = False):
         data = {'id': blk.to_dict()}
 

requirements.txt

@@ -1,3 +1,3 @@
 requests>=2.31.0
 setuptools>=65.5.1
-pytoniq-core>=0.1.10
+pytoniq-core>=0.1.32

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pytoniq",
-    version="0.1.34",
+    version="0.1.35",
     author="Maksim Kurbatov",
     author_email="[email protected]",
     description="TON Blockchain SDK",
@@ -22,8 +22,11 @@
     python_requires='>=3.9',
     py_modules=["pytoniq"],
     install_requires=[
-        "pytoniq-core>=0.1.31",
+        "pytoniq-core>=0.1.32",
         "requests>=2.31.0",
         "setuptools>=65.5.1",
-    ]
+    ],
+    extras_require={
+        'tvm': ['pytvm>=0.0.11'],
+    }
 )

tests/test_liteclient.py

@@ -61,3 +61,6 @@ async def test_get_method(client: LiteClient):
     result = await client.run_get_method(address='EQBvW8Z5huBkMJYdnfAEM5JqTNkuWX3diqYENkWsIL0XggGG', method='seqno',
                                          stack=[])
     assert isinstance(result[0], int)
+    result2 = await client.run_get_method_local(address='EQBvW8Z5huBkMJYdnfAEM5JqTNkuWX3diqYENkWsIL0XggGG', method='seqno',
+                                                stack=[])
+    assert result2 == result