GH-631: Gossip Storm Avoidance (#196)

* GH-631: get the skeleton ready

* GH-631: add a working recursive method for computing patch

* GH-631: add test for computing patch; not working yet

* GH-631: improve the test for the compute_patch function and make it working

* GH-631: refactor the recursive function to compute patch

* GH-631: refactor the conditionals inside the recursion

* GH-631: refactor the compute_patch() to use match

* GH-631: compute_patch() should search for neighbors in it's database for the root node

* GH-631: improve the solution for the compute_patch()

* GH-631: minor code cleanup

* GH-631: add one more test to make compute_patch() robust

* GH-631: add one more test for compute_patch()

* GH-631: reduce the number of recursion calls and minize hashset size

* GH-631: add test to check if the standard gossip handler ignores gossips that are outside the patch

* GH-631: make the test work

* GH-631: fix the test proper_standard_gossip_is_matched_and_handled

* GH-631: fix the tests inside gossip_acceptor.rs

* GH-631: refactor the compute_patch()

* GH-631: remove compiler warnings

* GH-631: filter agrs before calling the function that'll add introductory node

* GH-631: only use references to build hashmap from agr

* GH-631: use constant for computing patch

* GH-631: remove redundant code

* GH-631: remove clippy warnings

* GH-631: rename test to can_retrieve_all_full_and_half_neighbors()

* GH-631: remove contract test for eth ropsten

* GH-631: modify extract_node_reference()

* GH-631: Did more fixing on the blockchain multinode test

* GH-631: format the code

* GH-600: Workaround multinode test libc failure

* GH-631: Review 1 (#202)

* GH-631: refactor the compute_patch()

* GH-631: use node record instead of database

* GH-631: rename the some_node into not_a_neighbor

* GH-631: create an outboard function for creating patch

* GH-631: make a function for filtering agrs

* GH-631: remove unused functions

* GH-631: refactor test in gossip_acceptor.rs

* GH-631: minor changes in gossip_acceptor.rs

* GH-631: write test gossip_acceptor_filters_out_the_node_addr_of_incoming_gossip; not working yet

* GH-631: remove the test that you've recently written

* GH-631: Review 1 changes

* GH-631: remove clippy warnings

* GH-633: change the version number of libc from 2.36-4 to 2.36-6

* Trigger Build

* GH-631: An attempt to fix multinode test provided_and_consumed_services_are_recorded_in_databases() by using duration

* GH-631: remove the clippy allow statement for the compute_patch_recursive()

Co-authored-by: Dan Wiebe <[email protected]>
Co-authored-by: masqrauder <[email protected]>

multinode_integration_tests/docker/Dockerfile

@@ -1,10 +1,11 @@
 # Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved.
 #FROM debian:stable-slim
 #FROM debian:buster-slim
-FROM debian:bullseye-slim
+#FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 RUN apt-get update && \
-    apt-get install -y libc6 && \
+    apt-get install -y libc6=2.36-6 && \
 # These lines are commented out because for some reason the installation of iptables-persistent hangs forever on
 # bullseye-slim. Its absence means that the NodeStartupConfigBuilder::open_firewall_port() function won't work, but
 # at the time of this comment it's used in only one place in a way that doesn't have any value. So we decided to

multinode_integration_tests/src/masq_real_node.rs

@@ -1144,39 +1144,42 @@ impl MASQRealNode {
     }
 
     fn extract_node_reference(name: &str) -> Result<NodeReference, String> {
-        let regex = Self::descriptor_regex();
+        let descriptor_regex = Self::descriptor_regex();
         let mut retries_left = 25;
         loop {
+            if retries_left <= 0 {
+                return Err(format!("Node {} never started", name));
+            }
+            retries_left -= 1;
             println!("Checking for {} startup", name);
-            thread::sleep(Duration::from_millis(100));
-            let output = Self::exec_command_on_container_and_wait(
+            thread::sleep(Duration::from_millis(250));
+            match Self::exec_command_on_container_and_wait(
                 name,
                 vec![
                     "cat",
                     &format!("{}/{}", DATA_DIRECTORY, CURRENT_LOGFILE_NAME),
                 ],
-            )
-            .unwrap_or_else(|e| {
-                panic!(
-                    "Failed to read {}/{}: {}",
-                    DATA_DIRECTORY, CURRENT_LOGFILE_NAME, e
-                )
-            });
-            match regex.captures(output.as_str()) {
-                Some(captures) => {
-                    let node_reference =
-                        NodeReference::from_str(captures.get(1).unwrap().as_str()).unwrap();
-                    println!("{} startup detected at {}", name, node_reference);
-                    return Ok(node_reference);
-                }
-                None => {
-                    if retries_left <= 0 {
-                        return Err(format!("Node {} never started:\n{}", name, output));
+            ) {
+                Ok(output) => {
+                    if let Some(captures) = descriptor_regex.captures(output.as_str()) {
+                        let node_reference =
+                            NodeReference::from_str(captures.get(1).unwrap().as_str()).unwrap();
+                        println!("{} startup detected at {}", name, node_reference);
+                        return Ok(node_reference);
                     } else {
-                        retries_left -= 1;
+                        println!(
+                            "No local descriptor for {} in logfile yet\n{}",
+                            name, output
+                        )
                     }
                 }
-            }
+                Err(e) => {
+                    println!(
+                        "Failed to cat logfile for {} at {}/{}: {}",
+                        name, DATA_DIRECTORY, CURRENT_LOGFILE_NAME, e
+                    );
+                }
+            };
         }
     }
 }

multinode_integration_tests/tests/blockchain_interaction_test.rs

@@ -136,7 +136,7 @@ fn debtors_are_credited_once_but_not_twice() {
 }
 
 #[test]
-fn blockchain_bridge_logs_when_started() {
+fn blockchain_bridge_starts_properly_on_bootstrap() {
     let mut cluster = MASQNodeCluster::start().unwrap();
     let private_key = "0011223300112233001122330011223300112233001122330011223300112233";
     let subject = cluster.start_real_node(

multinode_integration_tests/tests/bookkeeping_test.rs

@@ -23,7 +23,7 @@ fn provided_and_consumed_services_are_recorded_in_databases() {
         .map(|_| start_real_node(&mut cluster, originating_node.node_reference()))
         .collect::<Vec<MASQRealNode>>();
 
-    thread::sleep(Duration::from_millis(10_000));
+    thread::sleep(Duration::from_secs(10));
 
     let mut client = originating_node.make_client(8080);
     let request = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n".as_bytes();
@@ -39,6 +39,9 @@ fn provided_and_consumed_services_are_recorded_in_databases() {
     // get all payables from originating node
     let payables = non_pending_payables(&originating_node);
 
+    // Waiting until the serving nodes have finished accruing their receivables
+    thread::sleep(Duration::from_secs(2));
+
     // get all receivables from all other nodes
     let receivable_balances = non_originating_nodes
         .iter()

node/src/neighborhood/gossip.rs

@@ -25,7 +25,7 @@ use std::net::{IpAddr, SocketAddr};
 pub struct GossipNodeRecord {
     pub signed_data: PlainData,
     pub signature: CryptData,
-    pub node_addr_opt: Option<NodeAddr>, // Only for use in introductions
+    pub node_addr_opt: Option<NodeAddr>,
 }
 
 impl Debug for GossipNodeRecord {