Cache dev

Cargo.toml

@@ -30,7 +30,7 @@ libc = "0.2"
 log = "0.4"
 miniz_oxide = "0.6"
 mio = { version = "1", features = ["os-poll", "os-ext", "net"] }
-openssl = "=0.10.66"
+openssl = "=0.10.70"
 paste = "1.0"
 rustls = "0.21"
 rustls-native-certs = "0.6"

INSTALL.md

@@ -0,0 +1,216 @@
+## Building from source
+```sh
+git clone -b cache-dev https://github.com/radiumb/pushpin.git
+```
+
+### Dependencies
+---
+If you’re using a recent version of Debian or Ubuntu, then all dependencies can be installed via package management:
+```sh
+sudo apt-get install make pkg-config rustc cargo g++ libzmq3-dev libssl-dev libboost-dev qtbase5-dev
+```
+If you’re on macOS, then all dependencies can be installed via Homebrew:
+```sh
+brew install pkg-config rust zeromq boost qt
+```
+### Compiling
+---
+Simply run make:
+```sh
+make
+```
+Optionally, you can install:
+```sh
+sudo make install
+```
+
+## Running
+Running on Debian and Ubuntu
+```sh
+sudo pushpin --loglevel=2,proxy:3 --logfile=/var/log/pushpin/pushpin.log
+```
+loglevel=2 : only show warnings, infos. 3 : show debug messages
+
+## Configuration
+Pushpin has two primary configuration files: `pushpin.conf` and routes. The `pushpin.conf` file refers to the routes file via the routesfile field:
+```sh
+[proxy]
+routesfile=routes
+```
+If a relative path is used, then the file is looked up relative to the location of `pushpin.conf`.
+```sh
+sudo cp examples/config/pushpin.conf /usr/local/etc/pushpin/pushpin.conf
+```
+```sh
+sudo cp examples/config/routes /usr/local/etc/pushpin/routes
+```
+### pushpin.conf file
+For general options, please refer to [this](https://pushpin.org/docs/configuration/#pushpinconf-file).
+Options for cache are added into this file.
+* <a name="cache_enable"></a>cache enable flag (false=disable, true=enable)
+	```
+	cache_enable=true
+	```
+#### backends
+* <a name="http_backend_urls"></a>http url path list of backends when failed to get response
+	```
+	http_backend_urls=http://localhost:7999/ws1,http://localhost:7999/ws2,http://localhost:7999/ws3 ...
+	```
+* <a name="ws_backend_urls"></a>websocket url paths for cache clients to connect to pushpin
+	```
+	ws_backend_urls=ws://localhost:7999/ws1,ws://localhost:7999/ws2,ws://localhost:7999/ws3 ...
+	```
+#### cache methods
+* <a name="cache_methods"></a>cache methods
+	```
+	ws_cache_methods=author_hasKey,author_hasSessionKeys,...
+	ws_subscribe_methods=beefy_subscribeJustifications,chain_subscribeAllHeads,...
+	(* means ALL methods, ex : ws_cache_methods=*)
+	```
+#### cache refresh
+* <a name="ws_never_timeout_methods"></a>never timeout/delete methods with params<br />
+	ref: List of methods that are not even delete/auto-refresh if it has the valid parameters
+	```
+	ws_never_timeout_methods=chain_getBlockHash,chain_getBlock,chain_getHeader,state_queryStorageAt,...
+	```
+* <a name="ws_refresh_shorter_methods"></a>cache auto-refresh shorter timeout methods<br />
+	ref: List of methods with shorter auto-refresh timeout periods (5 seconds)
+	```
+	ws_auto_refresh_longer_timeout_methods=chain_getBlockHash,chain_getHeader,state_getKeysPaged,state_queryStorageAt,...
+	```
+* <a name="ws_refresh_longer_methods"></a>cache auto-refresh longer timeout methods<br />
+	ref: List of methods with longer auto-refresh timeout periods (60 seconds)
+	```
+	ws_refresh_longer_methods=state_getMetadata,...
+	```
+* <a name="ws_refresh_unerase_methods"></a>cache auto-refresh no delete methods<br />
+	ref: List of methods that exist permanently in the cache list and do auto-refresh periodically
+	```
+	ws_refresh_unerase_methods=chain_getBlockHash,system_health,eth_getBalance,...
+	```
+* <a name="ws_refresh_exclude_methods"></a>cache no auto-refresh methods<br />
+	ref: List of methods that need to be cached but not auto refreshed
+	```
+	ws_refresh_exclude_methods=state_getMetadata,...
+	```
+* <a name="ws_refresh_passthrough_methods"></a>cache pass-through methods<br />
+	ref: List of methods that are not even cached but pass though to backend i.e. don't reject
+	```
+	ws_refresh_passthrough_methods=state_getStorage,...
+	```
+* <a name="ws_null_response_methods"></a>null response methods<br />
+	ref: List of methods that treat null responses as valid
+	```
+	ws_null_response_methods=chainHead_v1_unpin,...
+	```
+#### cache keys
+* <a name="ws_cache_key"></a>important fields in request, used as a key to identify cache items
+	```
+	ws_cache_key = $request_json_value["method"]+$user_defined["request_args"]+$request_json_pair["jsonrpc"]
+	request_args = $request_json_value["params"]
+	(ex : $request_json_value["method"]="author_hasKey", $request_json_pair["jsonrpc"]="\"jsonrpc\":\"2.0\"")
+	```
+* <a name="message_id_attribute"></a>field name is used as ID in Request or Response (to support multi-protocol in future)
+	```
+	message_id_attribute="id"
+	```
+* <a name="message_method_attribute"></a>field name is used as Method in Request or Response (to support multi-protocol in future)
+	```
+	message_method_attribute="method"
+	```
+* <a name="message_params_attribute"></a>field name is used as Params in Request (to support never_timeout_methods_with_params)
+	```
+	message_params_attribute="params"
+	```
+* <a name="message_error_attributes"></a>field names are used as Error in Response
+	```
+	message_error_attributes=error,fault,bug
+	```
+#### timeout of auto-refresh
+* <a name="ws_auto_refresh_cache_timeout_seconds"></a>cache timeout seconds (default 20)<br />
+	ref: Number of seconds to count auto-refresh timeout
+	```
+	ws_auto_refresh_cache_timeout_seconds=20
+	```
+* <a name="ws_auto_refresh_shorter_timeout_seconds"></a>cache shorter timeout seconds (default 10)<br />
+	ref: Number of seconds to count auto-refresh shorter timeout
+	```
+	ws_auto_refresh_shorter_timeout_seconds=10
+	```
+* <a name="ws_auto_refresh_longer_timeout_seconds"></a>cache longer timeout seconds (default 60)<br />
+	ref: Number of seconds to count auto-refresh longer timeout
+	```
+	ws_auto_refresh_longer_timeout_seconds=60
+	```
+* <a name="ws_auto_refresh_access_timeout_seconds"></a>cache auto-refresh access timeout seconds (default 30)<br />
+    ref: Number of seconds to count time out, be used to delete cache items that are not referenced by any client for this timeout period
+    ```
+    ws_auto_refresh_access_timeout_seconds=30
+    ```
+#### count of cache items
+* <a name="cache_item_max_count"></a>Maximum cache item count (default 3000)
+	```
+	cache_item_max_count=3000
+	```
+#### timeout
+* <a name="backend_switch_interval_seconds"></a>time seconds to switch another backend for null response (default 10)
+	```
+	backend_switch_interval_seconds=10
+	```
+#### error
+* <a name="message_error_attributes"></a>Fields in the response identifying errors
+	```
+	message_error_attributes=error,fault,bug
+	```
+#### prometheus
+* <a name="prometheus_restore_allow_seconds"></a>prometheus restore allow seconds (default 300)
+	```
+	prometheus_restore_allow_seconds=250
+	```
+#### redis
+* <a name="redis_enable"></a>redis enable flag (default false)
+	```
+	redis_enable=true
+	```
+* <a name="redis_host_addr"></a>redis server ip address (default 127.0.0.1)
+	```
+	redis_host_addr=127.0.0.1
+	```
+* <a name="redis_port"></a>redis server port number (default 6379)
+	```
+	redis_port=6379
+	```
+* <a name="redis_pool_count"></a>redis server port number (default 100)
+	```
+	redis_pool_count=100
+	```
+* <a name="redis_key_header"></a>Redis key prefix to identify each Pushpin instance when sharing Redis between multiple instances.
+	```
+	redis_key_header="pushpin1:"
+	```
+* <a name="replica_master_addr"></a>IP address of the replication master. If this is set to a valid IP address, Redis will act as a replica and attempt to connect to the specified master.
+	```
+	replica_master_addr=""
+	```
+* <a name="replica_master_port"></a>replication master port number (default 6379). Specifies the port Redis should use to connect to the master.
+	```
+	replica_master_port=6379
+	```
+#### Counts
+* <a name="prometheus_count"></a>groups for promethus count
+	```
+	ws_count_groups=ws_key_group,ws_index_group,ws_block_group,ws_state_subscribe,ws_cache_candidates
+	ws_key_group=author_hasKey,author_insertKey,...
+	ws_block_group=chain_getBlock,chain_getBlockHash,...
+	ws_state_subscribe=state_subscribeRuntimeVersion,state_subscribeStorage,...
+	ws_cache_candidates=author_hasKey,author_hasSessionKeys,...
+	```
+
+## Appendix
+### Installing redis-server
+	```
+	sudo apt update
+	sudo apt install redis-server -y
+	sudo systemctl enable redis-server
+	sudo systemctl start redis-server
+	```

Makefile

@@ -1,11 +1,13 @@
-cargo_flags =
+# controlled leading whitespace, per the GNU make manual
+nullstring :=
+space := $(nullstring) # end of the line
+
 ifdef RELEASE
-cargo_flags += --offline --locked --release
+cargo_flags = $(space)--offline --locked --release
 endif
 
-cargo_toolchain =
 ifdef TOOLCHAIN
-cargo_toolchain += +$(TOOLCHAIN)
+cargo_toolchain = $(space)+$(TOOLCHAIN)
 endif
 
 all: postbuild

examples/config/pushpin.conf

@@ -37,7 +37,7 @@ logdir=log
 log_level=2
 
 # client full request header must fit in this buffer
-client_buffer_size=8192
+client_buffer_size=81920
 
 # maximum number of client connections
 client_maxconn=50000
@@ -108,12 +108,17 @@ sockjs_url=http://cdn.jsdelivr.net/sockjs/0.3.4/sockjs.min.js
 # pushpin will output a log message when a new version is available. report
 # mode helps the pushpin project build credibility, so please enable it if you
 # enjoy this software :)
-updates_check=report
+updates_check=off
 
 # use this field to identify your organization in updates requests. if left
 # blank, updates requests will be anonymous
 organization_name=
 
+# prometheus report
+prometheus_port=9001
+
+workers=1
+
 
 [handler]
 # ipc permissions (octal)
@@ -171,3 +176,72 @@ stats_report_interval=10
 
 # stats output format
 stats_format=tnetstring
+
+
+[cache]
+cache_enable=true
+
+# backends
+http_backend_urls=http://localhost:7999/ws1,http://localhost:7999/ws2,http://localhost:7999/ws3,http://localhost:7999/ws4,http://localhost:7999/ws5
+ws_backend_urls=ws://localhost:7999/ws1,ws://localhost:7999/ws2,ws://localhost:7999/ws3,ws://localhost:7999/ws4,ws://localhost:7999/ws5
+
+# cache methods
+ws_cache_methods=*
+#author_hasKey,author_hasSessionKeys,author_insertKey,author_pendingExtrinsics,author_removeExtrinsic,author_rotateKeys,author_submitExtrinsic,babe_epochAuthorship,beefy_getFinalizedHead,chain_getBlock,chain_getBlockHash,chain_getFinalizedHead,chain_getHeader,childstate_getKeys,childstate_getKeysPaged,childstate_getStorage,childstate_getStorageEntries,childstate_getStorageHash,childstate_getStorageSize,contracts_call,contracts_getStorage,contracts_instantiate,contracts_rentProjection,contracts_upload_code,dev_getBlockStats,engine_createBlock,engine_finalizeBlock,eth_accounts,eth_blockNumber,eth_call,eth_chainId,eth_coinbase,eth_estimateGas,eth_feeHistory,eth_gasPrice,eth_getBalance,eth_getBlockByHash,eth_getBlockByNumber,eth_getBlockTransactionCountByHash,eth_getBlockTransactionCountByNumber,eth_getCode,eth_getFilterChanges,eth_getFilterLogs,eth_getLogs,eth_getProof,eth_getStorageAt,eth_getTransactionByBlockHashAndIndex,eth_getTransactionByBlockNumberAndIndex,eth_getTransactionByHash,eth_getTransactionCount,eth_getTransactionReceipt,eth_getUncleByBlockHashAndIndex,eth_getUncleCountByBlockHash,eth_getUncleCountByBlockNumber,eth_getWork,eth_hashrate,eth_maxPriorityFeePerGas,eth_mining,eth_newBlockFilter,eth_newFilter,eth_newPendingTransactionFilter,eth_protocolVersion,eth_sendRawTransaction,eth_sendTransaction,eth_submitHashrate,eth_submitWork,eth_syncing,eth_uninstallFilter,net_listening,net_version,web3_clientVersion,web3_sha3,grandpa_proveFinality,grandpa_roundState,mmr_generateBatchProof,mmr_generateProof,offchain_localStorageGet,offchain_localStorageSet,payment_queryFeeDetails,payment_queryInfo,rpc_methods,state_call,state_getChildKeys,state_getChildReadProof,state_getChildStorage,state_getChildStorageHash,state_getChildStorageSize,state_getKeys,state_getKeysPaged,state_getMetadata,state_getPairs,state_getReadProof,state_getRuntimeVersion,state_getStorage,state_getStorageHash,state_getStorageSize,state_queryStorage,state_queryStorageAt,state_traceBlock,state_trieMigrationStatus,sync_state_genSyncSpec,system_accountNextIndex,system_addLogFilter,system_addReservedPeer,system_chain,system_chainType,system_dryRun,system_health,system_localListenAddresses,system_localPeerId,system_name,system_networkState,system_nodeRoles,system_peers,system_properties,system_removeReservedPeer,system_reservedPeers,system_resetLogFilter,system_syncState,system_version
+ws_subscribe_methods=author_submitAndWatchExtrinsic+author_unwatchExtrinsic,beefy_subscribeJustifications+beefy_unsubscribeJustifications,chain_subscribeAllHeads+chain_unsubscribeAllHeads,chain_subscribeFinalisedHeads+chain_unsubscribeFinalisedHeads,chain_subscribeFinalizedHeads+chain_unsubscribeFinalizedHeads,chain_subscribeNewHead+chain_unsubscribeNewHead,chain_subscribeNewHeads+chain_unsubscribeNewHeads,chain_subscribeRuntimeVersion+chain_unsubscribeRuntimeVersion,grandpa_subscribeJustifications+grandpa_unsubscribeJustifications,state_subscribeRuntimeVersion+state_unsubscribeRuntimeVersion,state_subscribeStorage+state_unsubscribeStorage,transaction_unstable_submitAndWatch+transaction_unstable_unwatch,chainHead_v1_follow+chainHead_v1_unfollow
+
+# cache refresh
+ws_never_timeout_methods=chain_getBlockHash,chain_getBlock,chain_getHeader,state_queryStorageAt
+ws_refresh_shorter_methods=chain_getBlockHash,chain_getHeader,state_getKeysPaged,state_queryStorageAt
+ws_refresh_longer_methods=state_getMetadata
+ws_refresh_unerase_methods=chain_getBlockHash,system_health,eth_getBalance
+ws_refresh_exclude_methods=state_getMetadata
+ws_refresh_passthrough_methods=state_getStorage
+ws_null_response_methods=chainHead_v1_unpin
+
+# cache key
+ws_cache_key = $request_json_value["method"] + $user_defined["request_args"]+$request_json_pair["jsonrpc"]
+request_args = $request_json_value["params"]
+message_id_attribute="id"
+message_method_attribute="method"
+message_params_attribute="params"
+
+# auto-refresh. timeout
+# cache timeout seconds (default 20)
+ws_auto_refresh_cache_timeout_seconds=20
+# cache shorter timeout seconds (default 10)
+ws_auto_refresh_shorter_cache_timeout_seconds=10
+# cache longer timeout seconds (default 60)
+ws_auto_refresh_longer_timeout_seconds=60
+# cache auto-refresh access timeout seconds (default 30)
+ws_auto_refresh_access_timeout_seconds=30
+
+# Maximum cache item count (default 3000)
+cache_item_max_count=3000
+
+# time seconds to retry another backend for null response (default 10)
+backend_switch_interval_seconds=10
+
+# error
+message_error_attributes=error,fault,bug
+
+# prometheus restore allow seconds (default 300)
+prometheus_restore_allow_seconds=250
+
+# redis
+redis_enable=false
+redis_host_addr=127.0.0.1
+redis_port=6379
+redis_pool_count=100
+redis_key_header="pushpin1:"
+replica_master_addr="107.155.71.83"
+replica_master_port=6379
+
+# groups to count
+ws_count_groups=ws_key_group,ws_index_group,ws_block_group,ws_chain_group,ws_state_subscribe,ws_cache_candidates
+ws_key_group=author_hasKey,author_insertKey,childstate_getKeys,childstate_getKeysPaged,state_getChildKeys,state_getKeys,
+ws_index_group=eth_getTransactionByBlockHashAndIndex,eth_getTransactionByBlockNumberAndIndex,eth_getUncleByBlockHashAndIndex,eth_getUncleByBlockNumberAndIndex,system_accountNextIndex
+ws_block_group=chain_getBlock,chain_getBlockHash,dev_getBlockStats,engine_createBlock,engine_finalizeBlock,eth_blockNumber,eth_getBlockByHash,eth_getBlockByNumber,eth_getBlockTransactionCountByHash,eth_getBlockTransactionCountByNumber,eth_getTransactionByBlockHashAndIndex
+ws_chain_group=chain_getBlock,chain_getBlockHash,chain_getFinalizedHead,chain_getHeader,chain_subscribeAllHeads,chain_subscribeFinalizedHeads,chain_subscribeNewHead
+ws_state_subscribe=state_subscribeRuntimeVersion,state_subscribeStorage
+ws_cache_candidates=author_hasKey,author_hasSessionKeys,author_insertKey,author_pendingExtrinsics,author_removeExtrinsic,author_rotateKeys,author_submitExtrinsic,babe_epochAuthorship,beefy_getFinalizedHead,chain_getBlock,chain_getBlockHash,chain_getFinalizedHead,chain_getHeader,childstate_getKeys,childstate_getKeysPaged,childstate_getStorage,childstate_getStorageEntries,childstate_getStorageHash,childstate_getStorageSize,contracts_call,contracts_getStorage,contracts_instantiate,contracts_rentProjection,contracts_upload_code,dev_getBlockStats,engine_createBlock,engine_finalizeBlock,eth_accounts,eth_blockNumber,eth_call,eth_chainId,eth_coinbase,eth_estimateGas,eth_feeHistory,eth_gasPrice,eth_getBalance,eth_getBlockByHash,eth_getBlockByNumber,eth_getBlockTransactionCountByHash,eth_getBlockTransactionCountByNumber,eth_getCode,eth_getFilterChanges,eth_getFilterLogs,eth_getLogs,eth_getProof,eth_getStorageAt,eth_getTransactionByBlockHashAndIndex,eth_getTransactionByBlockNumberAndIndex,eth_getTransactionByHash,eth_getTransactionCount,eth_getTransactionReceipt,eth_getUncleByBlockHashAndIndex,eth_getUncleCountByBlockHash,eth_getUncleCountByBlockNumber,eth_getWork,eth_hashrate,eth_maxPriorityFeePerGas,eth_mining,eth_newBlockFilter,eth_newFilter,eth_newPendingTransactionFilter,eth_protocolVersion,eth_sendRawTransaction,eth_sendTransaction,eth_submitHashrate,eth_submitWork,eth_syncing,eth_uninstallFilter,net_listening,net_version,web3_clientVersion,web3_sha3,grandpa_proveFinality,grandpa_roundState,mmr_generateBatchProof,mmr_generateProof,offchain_localStorageGet,offchain_localStorageSet,payment_queryFeeDetails,payment_queryInfo,rpc_methods,state_call,state_getChildKeys,state_getChildReadProof,state_getChildStorage,state_getChildStorageHash,state_getChildStorageSize,state_getKeys,state_getKeysPaged,state_getMetadata,state_getPairs,state_getReadProof,state_getRuntimeVersion,state_getStorage,state_getStorageHash,state_getStorageSize,state_queryStorage,state_queryStorageAt,state_traceBlock,state_trieMigrationStatus,sync_state_genSyncSpec,system_accountNextIndex,system_addLogFilter,system_addReservedPeer,system_chain,system_chainType,system_dryRun,system_health,system_localListenAddresses,system_localPeerId,system_name,system_networkState,system_nodeRoles,system_peers,system_properties,system_removeReservedPeer,system_reservedPeers,system_resetLogFilter,system_syncState,system_version,state_subscribeStorage,state_subscribeRuntimeVersion,chain_subscribeNewHead,chain_subscribeFinalizedHeads

examples/config/routes

@@ -1 +1,2 @@
-* test
+*,proto=ws,no_grip,debug localhost:8999
+*,proto=http,no_grip,debug localhost:8999

src/connmgr/batch.rs

@@ -147,7 +147,7 @@ impl Batch {
         self.nodes.remove(key.nkey);
     }
 
-    pub fn take_group<'a, 'b: 'a, F>(&'a mut self, get_id: F) -> Option<BatchGroup>
+    pub fn take_group<'a, 'b: 'a, F>(&'a mut self, get_id: F) -> Option<BatchGroup<'a, 'b>>
     where
         F: Fn(usize) -> Option<(&'b [u8], u32)>,
     {