fix(consensus)!: request/response protocol for foreign proposals + fi…

…xes (tari-project#1215)

Description
---
- fix(consensus)!: request/response protocol for foreign proposals +
fixes
- fix: last vote not to be included in NEWVIEWs 
- fix: transactions stuck in pool and never proposed in some cases
- feat: add VN button to swarm
- fix: pending transactions can be continued between epochs
- fix: lock conflicts are removed when rejected(no vote) block is
deleted
- feat: send NEWVIEW as soon as NOVOTE is decided
- fix: invalid dummy blocks when leader failure occurs after genesis
block
- tests: new dummy tests
- tests: uncomment previously flaky eviction cucumber
- fix: validate qc for newview

Motivation and Context
---

Fixes tari-project#1201 

How Has This Been Tested?
---
Manually, cucumber and new tests. Tested num committees increasing from
1 to 2 in swarm

What process can a PR reviewer use to test or verify this change?
---


Breaking Changes
---

- [ ] None
- [x] Requires data directory to be deleted
- [ ] Other - Please specify

applications/tari_swarm_daemon/src/webserver/rpc/validator_nodes.rs

@@ -8,15 +8,15 @@ use serde::{Deserialize, Serialize};
 
 use crate::{config::InstanceType, process_manager::InstanceId, webserver::context::HandlerContext};
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct ListValidatorNodesRequest {}
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize)]
 pub struct ListValidatorNodesResponse {
     pub nodes: Vec<ValidatorNodeInfo>,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize)]
 pub struct ValidatorNodeInfo {
     pub instance_id: InstanceId,
     pub name: String,
@@ -55,13 +55,14 @@ pub async fn list(
     Ok(ListValidatorNodesResponse { nodes })
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct ValidatorNodeCreateRequest {
-    pub name: String,
-    pub register: bool,
+    name: Option<String>,
+    register: bool,
+    mine: bool,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize)]
 pub struct ValidatorNodeCreateResponse {
     pub instance_id: InstanceId,
 }
@@ -70,14 +71,24 @@ pub async fn create(
     context: &HandlerContext,
     req: ValidatorNodeCreateRequest,
 ) -> Result<ValidatorNodeCreateResponse, anyhow::Error> {
-    let instance_id = context
-        .process_manager()
-        .create_instance(req.name, InstanceType::TariValidatorNode, HashMap::new())
+    let process_manager = context.process_manager();
+
+    let name = match req.name {
+        Some(name) => name,
+        None => {
+            let all_instances = process_manager.list_instances(None).await?;
+            format!("New VN-#{}", all_instances.len())
+        },
+    };
+    let instance_id = process_manager
+        .create_instance(name, InstanceType::TariValidatorNode, HashMap::new())
         .await?;
 
     if req.register {
-        context.process_manager().register_validator_node(instance_id).await?;
-        context.process_manager().mine_blocks(10).await?;
+        process_manager.register_validator_node(instance_id).await?;
+        if req.mine {
+            process_manager.mine_blocks(10).await?;
+        }
     }
 
     Ok(ValidatorNodeCreateResponse { instance_id })

applications/tari_swarm_daemon/webui/src/routes/Main.tsx

@@ -524,6 +524,10 @@ export default function Main() {
     jsonRpc("start_all", { instance_type: "TariValidatorNode" }).then(getInfo);
   };
 
+  const addValidatorNode = () => {
+    jsonRpc("add_validator_node", { name: null, register: true, mine: false }).then(getInfo);
+  };
+
   return (
     <div className="main">
       <button onClick={() => stopAll()}>Stop all VNs</button>
@@ -532,6 +536,7 @@ export default function Main() {
       <button onClick={() => setAutoRefresh(!autoRefresh)}>{autoRefresh && "Disable" || "Enable"} autorefresh
       </button>
       <button onClick={() => setHorizontal(!horizontal)}>Swap rows/columns</button>
+      <button onClick={() => addValidatorNode()}>Add VN</button>
       <div className="label">Base layer</div>
       <div className="infos">
         <MinotariNodes showLogs={showLogs} />

applications/tari_validator_node/src/json_rpc/handlers.rs

@@ -342,8 +342,9 @@ impl JsonRpcHandlers {
 
         let transaction = self
             .state_store
-            .with_read_tx(|tx| ExecutedTransaction::get(tx, &data.transaction_id))
-            .map_err(internal_error(answer_id))?;
+            .with_read_tx(|tx| ExecutedTransaction::get(tx, &data.transaction_id).optional())
+            .map_err(internal_error(answer_id))?
+            .ok_or_else(|| not_found(answer_id, format!("Transaction {} not found", data.transaction_id)))?;
 
         Ok(JsonRpcResponse::success(answer_id, GetTransactionResponse {
             transaction,
@@ -410,40 +411,25 @@ impl JsonRpcHandlers {
     pub async fn get_block(&self, value: JsonRpcExtractor) -> JrpcResult {
         let answer_id = value.get_answer_id();
         let data: GetBlockRequest = value.parse_params()?;
-        let tx = self.state_store.create_read_tx().map_err(internal_error(answer_id))?;
-        match Block::get(&tx, &data.block_id) {
-            Ok(block) => {
-                let res = GetBlockResponse { block };
-                Ok(JsonRpcResponse::success(answer_id, res))
-            },
-            Err(e) => Err(JsonRpcResponse::error(
-                answer_id,
-                JsonRpcError::new(
-                    JsonRpcErrorReason::InvalidParams,
-                    format!("Something went wrong: {}", e),
-                    json::Value::Null,
-                ),
-            )),
-        }
+        let block = self
+            .state_store
+            .with_read_tx(|tx| Block::get(tx, &data.block_id).optional())
+            .map_err(internal_error(answer_id))?
+            .ok_or_else(|| not_found(answer_id, format!("Block {} not found", data.block_id)))?;
+
+        let res = GetBlockResponse { block };
+        Ok(JsonRpcResponse::success(answer_id, res))
     }
 
     pub async fn get_blocks_count(&self, value: JsonRpcExtractor) -> JrpcResult {
         let answer_id = value.get_answer_id();
-        let tx = self.state_store.create_read_tx().map_err(internal_error(answer_id))?;
-        match Block::get_count(&tx) {
-            Ok(count) => {
-                let res = GetBlocksCountResponse { count };
-                Ok(JsonRpcResponse::success(answer_id, res))
-            },
-            Err(e) => Err(JsonRpcResponse::error(
-                answer_id,
-                JsonRpcError::new(
-                    JsonRpcErrorReason::InternalError,
-                    format!("Something went wrong: {}", e),
-                    json::Value::Null,
-                ),
-            )),
-        }
+        let count = self
+            .state_store
+            .with_read_tx(|tx| Block::get_count(tx))
+            .map_err(internal_error(answer_id))?;
+
+        let res = GetBlocksCountResponse { count };
+        Ok(JsonRpcResponse::success(answer_id, res))
     }
 
     pub async fn get_filtered_blocks_count(&self, value: JsonRpcExtractor) -> JrpcResult {

applications/tari_validator_node/src/json_rpc/server.rs

@@ -23,7 +23,7 @@
 use std::{net::SocketAddr, sync::Arc};
 
 use axum::{extract::Extension, routing::post, Router};
-use axum_jrpc::{JrpcResult, JsonRpcAnswer, JsonRpcExtractor};
+use axum_jrpc::{error::JsonRpcErrorReason, JrpcResult, JsonRpcAnswer, JsonRpcExtractor};
 use log::*;
 use tower_http::cors::CorsLayer;
 
@@ -111,8 +111,12 @@ async fn handler(Extension(handlers): Extension<Arc<JsonRpcHandlers>>, value: Js
                     serde_json::to_string_pretty(val).unwrap_or_else(|e| e.to_string())
                 );
             },
+            // Log application errors as debug as these are typically intentional
+            JsonRpcAnswer::Error(err) if matches!(err.error_reason(), JsonRpcErrorReason::ApplicationError(_)) => {
+                debug!(target: LOG_TARGET, "JSON-RPC: {}", err);
+            },
             JsonRpcAnswer::Error(err) => {
-                error!(target: LOG_TARGET, "🚨 JSON-RPC request failed: {}", err);
+                error!(target: LOG_TARGET, "JSON-RPC request failed: {}", err);
             },
         }
     }

applications/tari_validator_node/src/p2p/services/messaging/outbound.rs

@@ -102,7 +102,29 @@ impl<TMsgLogger: MessageLogger + Send> tari_consensus::traits::OutboundMessaging
         Ok(())
     }
 
-    async fn multicast<T>(&mut self, shard_group: ShardGroup, message: T) -> Result<(), OutboundMessagingError>
+    async fn multicast<T, I>(&mut self, addresses: I, message: T) -> Result<(), OutboundMessagingError>
+    where
+        I: IntoIterator<Item = Self::Addr> + Send,
+        T: Into<HotstuffMessage> + Send,
+    {
+        let message = message.into();
+
+        self.networking
+            .send_multicast(
+                addresses
+                    .into_iter()
+                    .filter(|addr| *addr != self.our_node_addr)
+                    .map(|addr| addr.as_peer_id())
+                    .collect::<Vec<_>>(),
+                proto::consensus::HotStuffMessage::from(&message),
+            )
+            .await
+            .map_err(OutboundMessagingError::from_error)?;
+
+        Ok(())
+    }
+
+    async fn broadcast<T>(&mut self, shard_group: ShardGroup, message: T) -> Result<(), OutboundMessagingError>
     where T: Into<HotstuffMessage> + Send {
         let message = message.into();
 

bindings/dist/index.d.ts

@@ -2,25 +2,25 @@ export * from "./types/AbortReason";
 export * from "./types/AccessRule";
 export * from "./types/Account";
 export * from "./types/Amount";
-export * from "./types/Arg";
 export * from "./types/ArgDef";
+export * from "./types/Arg";
 export * from "./types/AuthHook";
-export * from "./types/Block";
 export * from "./types/BlockHeader";
+export * from "./types/Block";
 export * from "./types/BucketId";
 export * from "./types/Claims";
 export * from "./types/Command";
-export * from "./types/Committee";
 export * from "./types/CommitteeInfo";
 export * from "./types/CommitteeShardInfo";
+export * from "./types/Committee";
 export * from "./types/ComponentAccessRules";
 export * from "./types/ComponentAddress";
 export * from "./types/ComponentBody";
 export * from "./types/ComponentHeader";
 export * from "./types/ComponentKey";
 export * from "./types/ConfidentialClaim";
-export * from "./types/ConfidentialOutput";
 export * from "./types/ConfidentialOutputStatement";
+export * from "./types/ConfidentialOutput";
 export * from "./types/ConfidentialStatement";
 export * from "./types/ConfidentialTransferInputSelection";
 export * from "./types/ConfidentialWithdrawProof";
@@ -32,12 +32,12 @@ export * from "./types/Era";
 export * from "./types/Event";
 export * from "./types/EvictNodeAtom";
 export * from "./types/Evidence";
-export * from "./types/ExecuteResult";
 export * from "./types/ExecutedTransaction";
+export * from "./types/ExecuteResult";
 export * from "./types/ExtraData";
 export * from "./types/FeeBreakdown";
-export * from "./types/FeeClaim";
 export * from "./types/FeeClaimAddress";
+export * from "./types/FeeClaim";
 export * from "./types/FeeCostBreakdown";
 export * from "./types/FeeReceipt";
 export * from "./types/FeeSource";
@@ -46,10 +46,10 @@ export * from "./types/ForeignProposalAtom";
 export * from "./types/FunctionDef";
 export * from "./types/IndexedValue";
 export * from "./types/IndexedWellKnownTypes";
-export * from "./types/Instruction";
 export * from "./types/InstructionResult";
-export * from "./types/JrpcPermission";
+export * from "./types/Instruction";
 export * from "./types/JrpcPermissions";
+export * from "./types/JrpcPermission";
 export * from "./types/LeaderFee";
 export * from "./types/LockFlag";
 export * from "./types/LogEntry";
@@ -58,14 +58,14 @@ export * from "./types/Metadata";
 export * from "./types/MintConfidentialOutputAtom";
 export * from "./types/NetworkCommitteeInfo";
 export * from "./types/NodeHeight";
-export * from "./types/NonFungible";
-export * from "./types/NonFungibleAddress";
 export * from "./types/NonFungibleAddressContents";
+export * from "./types/NonFungibleAddress";
 export * from "./types/NonFungibleContainer";
 export * from "./types/NonFungibleId";
-export * from "./types/NonFungibleIndex";
 export * from "./types/NonFungibleIndexAddress";
+export * from "./types/NonFungibleIndex";
 export * from "./types/NonFungibleToken";
+export * from "./types/NonFungible";
 export * from "./types/NumPreshards";
 export * from "./types/Ordering";
 export * from "./types/OwnerRule";
@@ -75,50 +75,50 @@ export * from "./types/QuorumCertificate";
 export * from "./types/QuorumDecision";
 export * from "./types/RejectReason";
 export * from "./types/RequireRule";
-export * from "./types/Resource";
 export * from "./types/ResourceAccessRules";
 export * from "./types/ResourceAddress";
 export * from "./types/ResourceContainer";
+export * from "./types/Resource";
 export * from "./types/ResourceType";
 export * from "./types/RestrictedAccessRule";
 export * from "./types/ResumeNodeAtom";
 export * from "./types/RuleRequirement";
-export * from "./types/Shard";
 export * from "./types/ShardEvidence";
-export * from "./types/ShardGroup";
 export * from "./types/ShardGroupEvidence";
-export * from "./types/Substate";
+export * from "./types/ShardGroup";
+export * from "./types/Shard";
 export * from "./types/SubstateAddress";
 export * from "./types/SubstateDestroyed";
 export * from "./types/SubstateDiff";
 export * from "./types/SubstateId";
 export * from "./types/SubstateLockType";
 export * from "./types/SubstateRecord";
-export * from "./types/SubstateRequirement";
 export * from "./types/SubstateRequirementLockIntent";
+export * from "./types/SubstateRequirement";
+export * from "./types/Substate";
 export * from "./types/SubstateType";
 export * from "./types/SubstateValue";
 export * from "./types/SuspendNodeAtom";
 export * from "./types/TemplateDef";
 export * from "./types/TemplateDefV1";
-export * from "./types/Transaction";
 export * from "./types/TransactionAtom";
 export * from "./types/TransactionPoolRecord";
 export * from "./types/TransactionPoolStage";
-export * from "./types/TransactionReceipt";
 export * from "./types/TransactionReceiptAddress";
+export * from "./types/TransactionReceipt";
 export * from "./types/TransactionResult";
 export * from "./types/TransactionSignature";
 export * from "./types/TransactionStatus";
+export * from "./types/Transaction";
 export * from "./types/Type";
-export * from "./types/UnclaimedConfidentialOutput";
 export * from "./types/UnclaimedConfidentialOutputAddress";
+export * from "./types/UnclaimedConfidentialOutput";
 export * from "./types/UnsignedTransaction";
 export * from "./types/ValidatorSignature";
-export * from "./types/Vault";
 export * from "./types/VaultId";
-export * from "./types/VersionedSubstateId";
+export * from "./types/Vault";
 export * from "./types/VersionedSubstateIdLockIntent";
+export * from "./types/VersionedSubstateId";
 export * from "./types/ViewableBalanceProof";
 export * from "./base-node-client";
 export * from "./tari-indexer-client";