[master] ZIL-5332-debug_accountRange

src/libData/AccountStore/AccountStore.cpp

@@ -19,6 +19,7 @@
 #include <boost/filesystem/operations.hpp>
 #include <regex>
 
+#include "common/Common.h"
 #include "libData/AccountStore/AccountStore.h"
 #include "libData/AccountStore/services/evm/EvmClient.h"
 #include "libScilla/ScillaClient.h"
@@ -787,8 +788,18 @@ bool AccountStore::UpdateStateTrie(const Address &address,
     return false;
   }
 
-  std::lock_guard<std::mutex> g(m_mutexTrie);
-  m_state.insert(DataConversion::StringToCharArray(address.hex()), rawBytes);
+  std::lock(m_mutexTrie, m_mutexCache);
+  std::lock_guard<std::mutex> lock1(m_mutexTrie, std::adopt_lock);
+  std::lock_guard<std::mutex> lock2(m_mutexCache, std::adopt_lock);
+
+
+  zbytes z = DataConversion::StringToCharArray(address.hex());
+  if(!m_state.contains(z)){
+    std::array<unsigned char, 40> arr;
+    std::copy(z.begin(), z.end(), arr.begin());
+    m_cache.push_back(arr);
+  }
+  m_state.insert(z, rawBytes);
 
   return true;
 }
@@ -843,3 +854,40 @@ void AccountStore::PrintAccountState() {
   AccountStoreBase::PrintAccountState();
   LOG_GENERAL(INFO, "State Root: " << GetStateRootHash());
 }
+
+void AccountStore::FillAddressCache(){
+  std::lock(m_mutexTrie, m_mutexDB, m_mutexCache);
+  std::lock_guard<std::mutex> lock1(m_mutexTrie, std::adopt_lock);
+  std::lock_guard<std::mutex> lock2(m_mutexDB, std::adopt_lock);
+  std::lock_guard<std::mutex> lock3(m_mutexCache, std::adopt_lock);
+
+  m_cache.clear();
+
+  for(auto it = m_state.begin(); it != m_state.end(); ++it){
+    std::pair<zbytesConstRef , zbytesConstRef >item = it.at();
+    std::array<zbyte, 40> arr;
+    std::copy(item.first.begin(), item.first.end(), arr.begin());
+    m_cache.push_back(arr);
+  }
+}
+
+void AccountStore::PrintAddressCache(){
+  std::lock_guard<std::mutex> g(m_mutexCache);
+  for (const std::array<zbyte, 40>& entry : m_cache) {
+    std::string address(entry.begin(), entry.end());
+    LOG_GENERAL(INFO, "Address: " << address);
+  }
+
+}  
+
+std::vector<std::array<zbyte, 40>> AccountStore::GetAccountAddresses(unsigned long pageNumber, unsigned long pageSize, bool &wasMore){
+  //TODO: Implement input sanitisation before locking
+  std::lock_guard<std::mutex> g(m_mutexCache);
+  auto start = pageNumber * pageSize >= m_cache.size() ? m_cache.end() : m_cache.begin() + (pageNumber * pageSize);
+  wasMore = (pageNumber + 1) * pageSize < m_cache.size();
+  auto end = wasMore ? m_cache.begin() + ((pageNumber + 1) * pageSize) : m_cache.end();
+
+  std::vector<std::array<zbyte, 40>> slice(end - start);
+  std::copy(start, end, slice.begin());
+  return slice;
+}

src/libData/AccountStore/AccountStore.h

@@ -23,9 +23,11 @@
 #include <set>
 #include <shared_mutex>
 #include <unordered_map>
+#include <vector>
 
 #include <Schnorr.h>
 #include "AccountStoreBase.h"
+#include "common/Common.h"
 #include "common/Constants.h"
 #include "common/Hashes.h"
 #include "depends/libTrie/TrieDB.h"
@@ -75,6 +77,10 @@ class AccountStore : public AccountStoreBase {
 
   rpc::UnixDomainSocketServer m_scillaIPCServerConnector;
 
+  //cache storing the addresses in m_state
+  std::vector<std::array<unsigned char,40>> m_cache;
+  std::mutex m_mutexCache;
+
   AccountStore();
   ~AccountStore();
 
@@ -243,6 +249,9 @@ class AccountStore : public AccountStoreBase {
   bool UpdateStateTrieAll();
 
   void PrintAccountState() override;
+  void FillAddressCache();
+  void PrintAddressCache();
+  std::vector<std::array<zbyte, 40>> GetAccountAddresses(unsigned long pageNumber, unsigned long pageSize, bool &wasMore);
 };
 
 #endif  // ZILLIQA_SRC_LIBDATA_ACCOUNTSTORE_ACCOUNTSTORE_H_

src/libServer/EthRpcMethods.cpp

@@ -236,6 +236,17 @@ void EthRpcMethods::Init(LookupServer *lookupServer) {
                          jsonrpc::JSON_STRING, NULL),
       &EthRpcMethods::GetEthMiningI);
 
+  m_lookupServer->bindAndAddExternalMethod(
+      jsonrpc::Procedure("debug_accountRange", jsonrpc::PARAMS_BY_POSITION,
+                         jsonrpc::JSON_STRING, "param01", jsonrpc::JSON_INTEGER, "param02", jsonrpc::JSON_INTEGER,
+                         NULL),
+      &EthRpcMethods::GetDebugAccountRangeI);
+
+  m_lookupServer->bindAndAddExternalMethod(
+      jsonrpc::Procedure("debug_printCache", jsonrpc::PARAMS_BY_POSITION,
+                         jsonrpc::JSON_STRING, NULL),
+      &EthRpcMethods::PrintCacheContentsI);
+
   m_lookupServer->bindAndAddExternalMethod(
       jsonrpc::Procedure("eth_coinbase", jsonrpc::PARAMS_BY_POSITION,
                          jsonrpc::JSON_STRING, NULL),
@@ -1188,6 +1199,58 @@ Json::Value EthRpcMethods::GetEthMining() {
   return Json::Value(false);
 }
 
+Json::Value EthRpcMethods::GetDebugAccountRange(unsigned long pageNumber, unsigned long pageSize) {
+  INC_CALLS(GetInvocationsCounter());
+
+  try {
+    unique_lock<shared_timed_mutex> lock(
+        AccountStore::GetInstance().GetPrimaryMutex());
+    //TODO: Remove this line
+    AccountStore::GetInstance().FillAddressCache();
+
+    //TODO: Add input sanitation
+    bool wasMore = false;
+    auto addresses = AccountStore::GetInstance().GetAccountAddresses(pageNumber,pageSize, wasMore);
+    Json::Value response = Json::objectValue;
+    Json::Value jsonAddresses = Json::arrayValue;
+
+    for (const std::array<zbyte, 40>& entry : addresses) {
+      std::string address(entry.begin(), entry.end());
+      jsonAddresses.append(address);
+    }
+
+    response["addresses"] = jsonAddresses;
+    response["wasMore"] = wasMore;
+
+    return response;
+  } catch (const JsonRpcException &je) {
+    LOG_GENERAL(INFO, "[Error] getting balance" << je.GetMessage());
+    throw je;
+  } catch (exception &e) {
+    LOG_GENERAL(INFO, "[Error]" << e.what());
+    throw JsonRpcException(ServerBase::RPC_MISC_ERROR, "Unable To Process");
+  }
+}
+
+Json::Value EthRpcMethods::PrintCacheContents() {
+  INC_CALLS(GetInvocationsCounter());
+
+  try {
+    unique_lock<shared_timed_mutex> lock(
+        AccountStore::GetInstance().GetPrimaryMutex());
+
+    AccountStore::GetInstance().PrintAddressCache();
+
+    return Json::Value("Printed contents in log.");
+  } catch (const JsonRpcException &je) {
+    LOG_GENERAL(INFO, "[Error] getting balance" << je.GetMessage());
+    throw je;
+  } catch (exception &e) {
+    LOG_GENERAL(INFO, "[Error]" << e.what());
+    throw JsonRpcException(ServerBase::RPC_MISC_ERROR, "Unable To Process");
+  }
+}
+
 std::string EthRpcMethods::GetEthCoinbase() {
   INC_CALLS(GetInvocationsCounter());
 

src/libServer/EthRpcMethods.h

@@ -24,6 +24,7 @@
 #include "libMediator/Mediator.h"
 #include "libMetrics/Api.h"
 #include "libUtils/GasConv.h"
+#include "libUtils/Logger.h"
 
 class LookupServer;
 
@@ -265,6 +266,18 @@ class EthRpcMethods {
     response = this->GetEthMining();
   }
 
+  inline virtual void GetDebugAccountRangeI(const Json::Value& request,
+                                    Json::Value& response) {
+    LOG_MARKER_CONTITIONAL(LOG_SC);
+    response = this->GetDebugAccountRange(request[0u].asInt64(), request[1u].asInt64());
+  }
+
+  inline virtual void PrintCacheContentsI(const Json::Value& /*request*/,
+                                    Json::Value& response) {
+    LOG_MARKER_CONTITIONAL(LOG_SC);
+    response = this->PrintCacheContents();
+  }
+
   /**
    * @brief Handles json rpc 2.0 request on method: eth_coinbase.
    * Returns the client coinbase address. The coinbase address is the
@@ -729,6 +742,8 @@ class EthRpcMethods {
   Json::Value GetEthUncleCount();
   Json::Value GetEthUncleBlock();
   Json::Value GetEthMining();
+  Json::Value GetDebugAccountRange(unsigned long pageNumber, unsigned long pageSize);
+  Json::Value PrintCacheContents();
   std::string GetEthCoinbase();
   Json::Value GetNetListening();
   std::string GetNetPeerCount();

src/libServer/IsolatedServer.cpp

@@ -219,6 +219,17 @@ void IsolatedServer::BindAllEvmMethods() {
                            jsonrpc::JSON_STRING, NULL),
         &LookupServer::GetEthMiningI);
 
+    AbstractServer<IsolatedServer>::bindAndAddMethod(
+        jsonrpc::Procedure("debug_accountRange", jsonrpc::PARAMS_BY_POSITION,
+                         jsonrpc::JSON_STRING, "param01", jsonrpc::JSON_INTEGER, "param02", jsonrpc::JSON_INTEGER,
+                         NULL),
+        &LookupServer::GetDebugAccountRangeI);
+
+    AbstractServer<IsolatedServer>::bindAndAddMethod(
+        jsonrpc::Procedure("debug_printCache", jsonrpc::PARAMS_BY_POSITION,
+                           jsonrpc::JSON_STRING, NULL),
+        &LookupServer::PrintCacheContentsI);
+
     AbstractServer<IsolatedServer>::bindAndAddMethod(
         jsonrpc::Procedure("eth_getUncleByBlockHashAndIndex",
                            jsonrpc::PARAMS_BY_POSITION, jsonrpc::JSON_ARRAY,

tests/EvmAcceptanceTests/test/otterscan/debug_accountRange.ts

@@ -0,0 +1,69 @@
+import {assert} from "chai";
+import sendJsonRpcRequest from "../../helpers/JsonRpcHelper";
+
+const METHOD = "debug_accountRange";
+describe(`Otterscan api tests: ${METHOD} #parallel`, function () {
+
+  it("We can get addresses from the node", async function () {
+
+    const PAGE_NUMBER = 0;
+    const PAGE_SIZE = 9;
+
+    await sendJsonRpcRequest(METHOD, 1, [PAGE_NUMBER,PAGE_SIZE], (result, status) => {
+      assert.equal(status, 200, "has status code");
+
+      let jsonObject = result.result;
+      assert.isNotEmpty(jsonObject.addresses, "Can find addresses of accounts on the node");
+    });
+  });
+
+  it("We can see constant ordering between requests", async function () {
+    const PAGE_NUMBER_1 = 0;
+    const PAGE_SIZE_1 = 3;
+
+    let shared_address : string;
+
+    await sendJsonRpcRequest(METHOD, 1, [PAGE_NUMBER_1,PAGE_SIZE_1], (result, status) => {
+      assert.equal(status, 200, "has status code");
+
+      let jsonObject = result.result;
+      shared_address = jsonObject.addresses[2];
+    });
+
+    const PAGE_NUMBER_2 = 1;
+    const PAGE_SIZE_2 = 2;
+
+    await sendJsonRpcRequest(METHOD, 1, [PAGE_NUMBER_2,PAGE_SIZE_2], (result, status) => {
+      assert.equal(status, 200, "has status code");
+
+      let jsonObject = result.result;
+      assert.strictEqual(jsonObject.addresses[0], shared_address, "There is consistent ordering of addresses between requests due to overlap");
+    });
+  });
+
+  it("We can view the first address and that there are more to be seen", async function () {
+    const PAGE_NUMBER = 0;
+    const PAGE_SIZE = 1;
+
+    await sendJsonRpcRequest(METHOD, 1, [PAGE_NUMBER,PAGE_SIZE], (result, status) => {
+      assert.equal(status, 200, "has status code");
+
+      let jsonObject = result.result;
+      assert.isNotEmpty(jsonObject.addresses, "Can find first addresses of accounts");
+      assert.isTrue(jsonObject.wasMore, "Can see that there are more addresses to be fetched");
+    });
+  });
+
+  it("We can reach the end of addresses and see that there are no more to be seen", async function () {
+    const PAGE_NUMBER = Number.MAX_SAFE_INTEGER;
+    const PAGE_SIZE = 1;
+
+    await sendJsonRpcRequest(METHOD, 1, [PAGE_NUMBER,PAGE_SIZE], (result, status) => {
+      assert.equal(status, 200, "has status code");
+
+      let jsonObject = result.result;
+      assert.isEmpty(jsonObject.addresses, "Can not find addresses outside range of cache");
+      assert.isFalse(jsonObject.wasMore, "Can see that there are no more addresses to be fetched");
+    });
+  });
+});