Bootstrap infrastucture code

- new config file options for bootstrap nodes
- new config file option for testnet
- update docker and docker compose to for easier running of boostrap
  nodes and testnet nodes

Author: PeterSurda

buildscripts/bootstrap/README.md

@@ -0,0 +1,2 @@
+This directory is for deploying bootstrap and test infrastructure for
+PyBitmessage.

buildscripts/bootstrap/docker-compose.yaml

@@ -0,0 +1,13 @@
+---
+version: "3"
+name: pybitmessage
+services:
+  bootstrap:
+    image: pybitmessage/bootstrap:latest
+    build: ..
+    env_file:
+      - /etc/pybitmessage.conf
+    ports:
+      - 127.0.0.1::8444
+    deploy:
+      replicas: $THREADS

buildscripts/bootstrap/start-loadbalancer.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+apt -y install curl jq ipvsadm libyajl2
+
+EXTIP=$(curl -s telnetmyip.com|jq -r .ip)
+if [ ! -e .env ]; then
+    THREADS=$(nproc --all)
+    PASSWORD=$(tr -dc a-zA-Z0-9 < /dev/urandom | head -c32 && echo)
+    cat > .env << EOF
+THREADS=$THREADS
+PASSWORD=$PASSWORD
+EOF
+else
+    . .env
+fi
+
+ipvsadm -C
+ipvsadm -A -t ${EXTIP}:8444 -s rr
+ipvsadm -A -t ${EXTIP}:8080 -s rr
+
+docker compose up -d
+
+CF=/etc/collectd/collectd.conf.d/curl_json.conf.new
+CF_LIVE=/etc/collectd/collectd.conf.d/curl_json.conf
+
+echo "LoadPlugin curl_json" > $CF
+echo "<Plugin curl_json>" >> $CF
+
+for i in `seq 1 $THREADS`; do
+    cont="pybitmessage-bootstrap-${i}"
+    IP=$(docker inspect -f
+        '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'
+        $cont 2>/dev/null)
+    [ -z "$IP" ] && continue
+    echo "Adding $IP"
+    ipvsadm -a -t ${EXTIP}:8444 -r ${IP}:8444 -m
+    ipvsadm -a -t ${EXTIP}:8080 -r ${IP}:8444 -m
+    INSTANCE=$(echo $cont|tr - _)
+    cat >> $CF << EOF
+  <URL "http://$IP:8442/">
+    Plugin "pybitmessagestatus"
+    Instance "$INSTANCE"
+    User "api"
+    Password "$PASSWORD"
+    Post "{\"jsonrpc\":\"2.0\",\"id\":\"id\",\"method\":\"clientStatus\",\"params\":[]}"
+    <Key "result/networkConnections">
+      Type "gauge"
+      Instance "networkconnections"
+    </Key>
+    <Key "result/numberOfPubkeysProcessed">
+      Type "counter"
+      Instance "numberofpubkeysprocessed"
+    </Key>
+    <Key "result/numberOfMessagesProcessed">
+      Type "counter"
+      Instance "numberofmessagesprocessed"
+    </Key>
+    <Key "result/numberOfBroadcastsProcessed">
+      Type "counter"
+      Instance "numberofbroadcastsprocessed"
+    </Key>
+  </URL>
+EOF
+done
+
+echo "</Plugin>" >> $CF
+
+if ! cmp -s $CF $CF_LIVE; then
+	mv $CF $CF_LIVE
+	systemctl restart collectd
+fi
+
+ipvsadm -l -n

packages/docker/launcher.sh

@@ -12,5 +12,33 @@ sed -i -e "s|\(apiinterface = \).*|\10\.0\.0\.0|g" \
     -e "s|\(apipassword = \).*|\1$APIPASS|g" \
     -e "s|apinotifypath = .*||g" ${BITMESSAGE_HOME}/keys.dat
 
+if [ -n "$PYBITMESAGE_BOOTSTRAP" -o -n "$PYBITMESSAGE_TESTNET" ]; then
+    echo "[bootstrap]" >> ${BITMESSAGE_HOME}/keys.dat
+fi
+
+if [ -n "$PYBITMESSAGE_BOOTSTRAP" ]; then
+    IP=$(hostname -i)
+    sed -i -e "s|\(apiinterface = \).*|\1$IP|g" \
+        -e "s|\(bind = \).*|\1$IP|g" \
+        ${BITMESSAGE_HOME}/keys.dat
+    echo <<(EOF) >> ${BITMESSAGE_HOME}/keys.dat
+idle_timeout = 60
+commands = True
+threads = True
+inv = True
+dup_ip = True
+(EOF)
+fi
+
+if [ -n "$PYBITMESSAGE_TESTNET" ]; then
+    IP=$(hostname -i)
+    sed -i -e "s|\(apiinterface = \).*|\1$IP|g" \
+        -e "s|\(bind = \).*|\1$IP|g" \
+        ${BITMESSAGE_HOME}/keys.dat
+    echo <<(EOF) >> ${BITMESSAGE_HOME}/keys.dat
+testnet = True
+(EOF)
+fi
+
 # Run
 exec pybitmessage "$@"

src/network/__init__.py

@@ -45,10 +45,13 @@ def start(config, state):  # pylint: disable=too-many-locals
 
     readKnownNodes()
     pool.connectToStream(1)
-    for thread in (
-        BMNetworkThread(pool), InvThread(pool), AddrThread(pool),
-        DownloadThread(pool), UploadThread(pool)
-    ):
+    threads_to_start = list()
+    threads_to_start.append(BMNetworkThread)
+    if not config.safeGetBoolean('bootstrap', 'threads'):
+        threads_to_start.extend([InvThread, AddrThread,
+                                 DownloadThread, UploadThread])
+    for thread_class in threads_to_start:
+        thread = thread_class(pool)
         thread.daemon = True
         thread.start()
 

src/network/bmproto.py

@@ -314,6 +314,8 @@ def decode_simple(self, char="v"):
 
     def bm_command_error(self):
         """Decode an error message and log it"""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         err_values = self.decode_payload_content("vvlsls")
         fatalStatus = err_values[0]
         # banTime = err_values[1]
@@ -330,6 +332,8 @@ def bm_command_getdata(self):
         If we have them and some other conditions are fulfilled,
         append them to the write queue.
         """
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         items = self.decode_payload_content("l32s")
         # skip?
         now = time.time()
@@ -366,14 +370,20 @@ def _command_inv(self, extend_dandelion_stem=False):
 
     def bm_command_inv(self):
         """Non-dandelion announce"""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         return self._command_inv(False)
 
     def bm_command_dinv(self):
         """Dandelion stem announce"""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         return self._command_inv(True)
 
     def bm_command_object(self):
         """Incoming object, process it"""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         objectOffset = self.payloadOffset
         nonce, expiresTime, objectType, version, streamNumber = \
             self.decode_payload_content("QQIvv")
@@ -443,6 +453,8 @@ def _decode_addr(self):
 
     def bm_command_addr(self):
         """Incoming addresses, process them"""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         # not using services
         for seenTime, stream, _, ip, port in self._decode_addr():
             ip = bytes(ip)
@@ -474,6 +486,8 @@ def bm_command_addr(self):
 
     def bm_command_portcheck(self):
         """Incoming port check request, queue it."""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         if self.isOutbound or self.portCheckRequested:
             return True
         self.portCheckRequested = True
@@ -483,6 +497,8 @@ def bm_command_portcheck(self):
 
     def bm_command_ping(self):
         """Incoming ping, respond to it."""
+        if config.safeGetBoolean('bootstrap', 'commands'):
+            return
         self.append_write_buf(protocol.CreatePacket('pong'))
         return True
 
@@ -603,7 +619,8 @@ def peerValidityChecks(self):
                 ' interest in streams.', self.destination)
             return False
         if self.pool.inboundConnections.get(
-                self.destination):
+                self.destination) and not \
+                config.safeGetBoolean('bootstrap', 'dup_ip'):
             try:
                 if not protocol.checkSocksIP(self.destination.host):
                     self.append_write_buf(protocol.assembleErrorMessage(

src/network/connectionpool.py

@@ -65,6 +65,12 @@ def __init__(self):
                 ' trustedpeer=<hostname>:<portnumber>'
             )
 
+        # default timeout 300, lower than 30 doesn't make much practical sense
+        self.idleTimeout = max([30,
+                                config.safeGetInt('bootstrap',
+                                                  'idle_timeout', 300)
+                                ])
+
     def __len__(self):
         return len(self.outboundConnections) + len(self.inboundConnections)
 
@@ -390,7 +396,7 @@ def loop(self):  # pylint: disable=too-many-branches,too-many-statements
         for i in self.connections():
             minTx = time.time() - 20
             if i.fullyEstablished:
-                minTx -= 300 - 20
+                minTx -= self.idleTimeout - 20
             if i.lastTx < minTx:
                 if i.fullyEstablished:
                     i.append_write_buf(protocol.CreatePacket('ping'))

src/network/knownnodes.py

@@ -222,8 +222,12 @@ def trimKnownNodes(recAddrStream=1):
 def dns():
     """Add DNS names to knownnodes"""
     for port in [8080, 8444]:
-        addKnownNode(
-            1, Peer('bootstrap%s.bitmessage.org' % port, port))
+        if config.safeGetBoolean('boostrap', 'testnet'):
+            addKnownNode(
+                1, Peer('bootstrap%s.testnet.bitmessage.org' % port, port))
+        else:
+            addKnownNode(
+                1, Peer('bootstrap%s.bitmessage.org' % port, port))
 
 
 def cleanupKnownNodes(pool):

src/network/tcp.py

@@ -168,8 +168,10 @@ def set_connection_fully_established(self):
             knownnodes.addKnownNode(
                 self.streams, self.destination, time.time())
             dandelion_ins.maybeAddStem(self, invQueue)
-        self.sendAddr()
-        self.sendBigInv()
+        if not config.safeGetBoolean('bootstrap', 'addr'):
+            self.sendAddr()
+        if not config.safeGetBoolean('bootstrap', 'inv'):
+            self.sendBigInv()
 
     def sendAddr(self):
         """Send a partial list of known addresses to peer."""
@@ -372,7 +374,8 @@ def set_connection_fully_established(self):
             """Only send addr here"""
             # pylint: disable=attribute-defined-outside-init
             self.fullyEstablished = True
-            self.sendAddr()
+            if not config.safeGetBoolean('bootstrap', 'addr'):
+                self.sendAddr()
 
         def handle_close(self):
             """

src/protocol.py

@@ -26,6 +26,8 @@
 
 # Network constants
 magic = 0xE9BEB4D9
+if config.safeGetBoolean('bootstrap', 'testnet'):
+    magic = 0xFB110907
 #: protocol specification says max 1000 addresses in one addr command
 MAX_ADDR_COUNT = 1000
 #: address is online if online less than this many seconds ago