cleanup testing and documentation across application (#107)

Testing for application/lib/registration.go and other application modules has gotten moderately out of date and needs tuning. This PR also moves towards cleaning up documentation and golang idioms to match golint standards. Finally this cleans up some spelling mistakes and missing field descriptions in config and documentation,

application/lib/registration_test.go

@@ -98,64 +98,6 @@ func TestRegistrationLookup(t *testing.T) {
 	}
 }
 
-func TestLivenessCheck(t *testing.T) {
-	phantomAddr := net.ParseIP("1.1.1.1")
-	reg := DecoyRegistration{
-		DarkDecoy: phantomAddr,
-	}
-
-	liveness, response := reg.PhantomIsLive()
-	if liveness != true {
-		t.Fatalf("Live host seen as non-responsive: %v\n", response)
-	}
-
-	// Is there any test address we know will never respond?
-	unroutableIP := net.ParseIP("127.0.0.2")
-	reg.DarkDecoy = unroutableIP
-
-	liveness, response = reg.PhantomIsLive()
-	if liveness == false {
-		t.Fatalf("Unroutable host seen as Live: %v\n", response)
-	}
-
-	// Is there any test address we know will never respond?
-	phantomV6 := net.ParseIP("2606:4700:4700::64")
-	reg.DarkDecoy = phantomV6
-
-	liveness, response = reg.PhantomIsLive()
-	if liveness != true {
-		t.Fatalf("Live V6 host seen as non-responsive: %v\n", response)
-	}
-
-	// Is there any test address we know will never respond?
-	unreachableV6 := net.ParseIP("2001:48a8:687f:1:1122::105")
-	reg.DarkDecoy = unreachableV6
-
-	liveness, response = reg.PhantomIsLive()
-	if liveness != false {
-		t.Fatalf("Non responsive V6 host seen as live: %v\n", response)
-	}
-}
-
-func TestLiveness(t *testing.T) {
-
-	liveness, response := phantomIsLive("1.1.1.1.:80")
-
-	if liveness != true {
-		t.Fatalf("Host is live, detected as NOT live: %v\n", response)
-	}
-
-	liveness, response = phantomIsLive("192.0.0.2:443")
-	if liveness != false {
-		t.Fatalf("Host is NOT live, detected as live: %v\n", response)
-	}
-
-	liveness, response = phantomIsLive("[2606:4700:4700::64]:443")
-	if liveness != true {
-		t.Fatalf("Host is live, detected as NOT live: %v\n", response)
-	}
-}
-
 func TestRegisterForDetectorOnce(t *testing.T) {
 	reg := DecoyRegistration{
 		DarkDecoy:        net.ParseIP("1.2.3.4"),

application/liveness/README.md

@@ -1,6 +1,17 @@
 # Conjure Liveness Module
 
-The liveness module is designed to keep track of all the live hosts in the network by scanning the internet on a regular basis. The module provides cached liveness detection and uncached liveness detection. Cached liveness detection stores live hosts in previous `PhantomIsLive` calls to improve performance, while uncached liveness detection directly visits an IP address to check its liveness status. The validity of cached liveness detection is tested in a week-long survey of the internet where we measured the change of liveness across the network. We observed a downward trend in the similarity of discovered hosts.
+The liveness module is designed to keep track of all the live hosts in the
+network by scanning the internet on a regular basis. The module provides cached
+liveness detection and uncached liveness detection. Cached liveness detection
+stores live hosts from previous `PhantomIsLive` calls to improve performance,
+non-responsive hosts and stale cache entries are re-scanned each time. Uncached
+liveness detection directly visits an IP address to check its liveness status on
+each call.
+
+The validity of cached liveness detection was tested in a week-long survey of
+the internet where we measured the change in liveness across the network. We
+observed a clear trend in the stability of discovered hosts over time and as
+such chose a 2 hour default cache period.
 
 ## Usage
 
@@ -14,4 +25,12 @@ To check if an IP address is live in the network, call `PhantomIsLive(addr strin
 |![Scanning Data(48 hours)](https://raw.githubusercontent.com/kkz13250/conjure_related/main/48_hours_scanning_plot.png)|
 | *Survey data for first 48 hours* |
 
-Percentages of the number of current live hosts divided by the number of cached live hosts(marked green) went down drastically in the first 24 hours of scanning the network which indicates that caching every discoverable live hosts is not an effective way to representy the current liveness status of the addresses in the network. Instead, we decided to cache invidual IP addresses that are passed to `PhantomIsLive` for checking its liveness status. Every cached address gets a timestamp when its liveness status is checked, cached addresses with expired timstamp will no longer be considered as live hosts by the module. Expiration duration can be set in the config file.
+Percentages of the number of current live hosts divided by the number of cached
+live hosts(marked green) went down drastically in the first 24 hours of scanning
+the network which indicates that caching every discoverable live hosts is not an
+effective way to represent the current liveness status of the addresses in the
+network over time. Instead, we decided to cache individual IP addresses that are passed to
+`PhantomIsLive` for checking its liveness status. Every cached address gets a
+timestamp when its liveness status is checked, cached addresses with expired
+timestamp will no longer be considered as live hosts by the module. Expiration
+duration can be set in the config file.

application/liveness/liveness.go

@@ -10,26 +10,35 @@ import (
 	"time"
 )
 
+// LivenessTester provides a generic interface for testing hosts in phantom
+// subnets for liveness. This prevents potential interference in connection
+// creation.
 type LivenessTester interface {
 	PhantomIsLive(addr string, port uint16) (bool, error)
 }
 
-type CacheElement struct {
+type cacheElement struct {
 	isLive     bool
 	cachedTime time.Time
 }
 
+// CachedLivenessTester implements LivenessTester interface with caching,
+// PhantomIsLive will check historical results first before using the network to
+// determine phantom liveness.
 type CachedLivenessTester struct {
-	ipCache             map[string]CacheElement
+	ipCache             map[string]cacheElement
 	signal              chan bool
 	cacheExpirationTime time.Duration
 }
 
+// UncachedLivenessTester implements LivenessTester interface without caching,
+// PhantomIsLive will always use the network to determine phantom liveness.
 type UncachedLivenessTester struct {
 }
 
+// Init parses cache expiry duration and initializes the Cache.
 func (blt *CachedLivenessTester) Init(expirationTime string) error {
-	blt.ipCache = make(map[string]CacheElement)
+	blt.ipCache = make(map[string]cacheElement)
 	blt.signal = make(chan bool)
 
 	convertedTime, err := time.ParseDuration(expirationTime)
@@ -41,10 +50,13 @@ func (blt *CachedLivenessTester) Init(expirationTime string) error {
 	return nil
 }
 
+// Stop end periodic scanning using running in separate goroutine. If periodic
+// scanning is not running this will do nothing.
 func (blt *CachedLivenessTester) Stop() {
 	blt.signal <- true
 }
 
+// ClearExpiredCache cleans out stale entries in the cache.
 func (blt *CachedLivenessTester) ClearExpiredCache() {
 	for ipAddr, status := range blt.ipCache {
 		if time.Since(status.cachedTime) > blt.cacheExpirationTime {
@@ -53,6 +65,9 @@ func (blt *CachedLivenessTester) ClearExpiredCache() {
 	}
 }
 
+// PeriodicScan uses zmap to populate the cache of a CachedLivenessTester.
+// Should be run as a goroutine as it may block for long periods of time while
+// scanning.
 func (blt *CachedLivenessTester) PeriodicScan(t string) {
 	os.Create("block_list.txt")
 	allowListAddr := os.Getenv("PHANTOM_SUBNET_LOCATION")
@@ -88,7 +103,7 @@ func (blt *CachedLivenessTester) PeriodicScan(t string) {
 			for _, ip := range records {
 				if ip[0] != "saddr" {
 					if _, ok := blt.ipCache[ip[0]]; !ok {
-						var val CacheElement
+						var val cacheElement
 						val.isLive = true
 						val.cachedTime = time.Now()
 						blt.ipCache[ip[0]] = val
@@ -121,6 +136,11 @@ func (blt *CachedLivenessTester) PeriodicScan(t string) {
 	}
 }
 
+// PhantomIsLive first checks the cached set of addressses for a fresh entry.
+// If one is available and the host was measured to be live this is returned
+// immediately and no network probes are sent. If the host was measured not
+// live, the entry is stale, or there is not entry then network probes are sent
+// and the result is then added to the cache.
 func (blt *CachedLivenessTester) PhantomIsLive(addr string, port uint16) (bool, error) {
 	// existing phantomIsLive() implementation
 	if status, ok := blt.ipCache[addr]; ok {
@@ -131,13 +151,17 @@ func (blt *CachedLivenessTester) PhantomIsLive(addr string, port uint16) (bool,
 		}
 	}
 	isLive, err := phantomIsLive(net.JoinHostPort(addr, strconv.Itoa(int(port))))
-	var val CacheElement
+	var val cacheElement
 	val.isLive = isLive
 	val.cachedTime = time.Now()
 	blt.ipCache[addr] = val
 	return isLive, err
 }
 
+// PhantomIsLive sends 4 TCP syn packets to determine if the host will respond
+// to traffic and potentially interfere with a connection if used as a phantom
+// address. Measurement results are uncached, meaning endpoints are re-scanned
+// every time.
 func (blt *UncachedLivenessTester) PhantomIsLive(addr string, port uint16) (bool, error) {
 	return phantomIsLive(net.JoinHostPort(addr, strconv.Itoa(int(port))))
 }
@@ -168,13 +192,13 @@ func phantomIsLive(address string) (bool, error) {
 	select {
 	case err := <-dialError:
 		if e, ok := err.(net.Error); ok && e.Timeout() {
-			return false, fmt.Errorf("Reached connection timeout")
+			return false, fmt.Errorf("reached connection timeout")
 		}
 		if err != nil {
 			return true, err
 		}
-		return true, fmt.Errorf("Phantom picked up the connection")
+		return true, fmt.Errorf("phantom picked up the connection")
 	default:
-		return false, fmt.Errorf("Reached statistical timeout %v", timeout)
+		return false, fmt.Errorf("reached statistical timeout %v", timeout)
 	}
 }

application/liveness/liveness_test.go

@@ -35,3 +35,66 @@ func TestStop(t *testing.T) {
 	go blt.PeriodicScan("Minutes")
 	blt.Stop()
 }
+
+func TestUncachedLiveness(t *testing.T) {
+
+	ult := UncachedLivenessTester{}
+
+	liveness, response := ult.PhantomIsLive("1.1.1.1.", 80)
+
+	if liveness != true {
+		t.Fatalf("Host is live, detected as NOT live: %v\n", response)
+	}
+
+	liveness, response = ult.PhantomIsLive("192.0.0.2", 443)
+	if liveness != false {
+		t.Fatalf("Host is NOT live, detected as live: %v\n", response)
+	}
+
+	liveness, response = ult.PhantomIsLive("2606:4700:4700::64", 443)
+	if liveness != true {
+		t.Fatalf("Host is live, detected as NOT live: %v\n", response)
+	}
+}
+
+func TestCachedLiveness(t *testing.T) {
+
+	clt := CachedLivenessTester{}
+	err := clt.Init("1h")
+	if err != nil {
+		t.Fatalf("failed to init cached liveness tester: %s", err)
+	}
+
+	liveness, response := clt.PhantomIsLive("1.1.1.1.", 80)
+	if liveness != true {
+		t.Fatalf("Host is live, detected as NOT live: %v\n", response)
+	}
+	if status, ok := clt.ipCache["1.1.1.1."]; !ok || status.isLive != true {
+		t.Fatalf("Host is live, but not cached as live")
+	}
+
+	liveness, response = clt.PhantomIsLive("192.0.0.2", 443)
+	if liveness != false {
+		t.Fatalf("Host is NOT live, detected as live: %v\n", response)
+	}
+	if status, ok := clt.ipCache["192.0.0.2"]; !ok || status.isLive != false {
+		t.Fatalf("Host is NOT live, but cached as live")
+	}
+
+	liveness, response = clt.PhantomIsLive("2606:4700:4700::64", 443)
+	if liveness != true {
+		t.Fatalf("Host is live, detected as NOT live: %v\n", response)
+	}
+	if status, ok := clt.ipCache["2606:4700:4700::64"]; !ok || status.isLive != true {
+		t.Fatalf("Host is not live, but cached as live")
+	}
+
+	// lookup for known live cached values should be fast since it doesn't go to network.
+	start := time.Now()
+	_, _ = clt.PhantomIsLive("2606:4700:4700::64", 443)
+	_, _ = clt.PhantomIsLive("1.1.1.1.", 80)
+	if time.Since(start) > time.Duration(1*time.Millisecond) {
+		t.Fatal("Lookup for cached live entries taking too long")
+	}
+
+}

registration-api/config.toml

@@ -23,5 +23,7 @@ station_pubkeys = [
 	"",
 ]
 
-# new
+
+# This field specifies the generation number that the Bidirectional API 
+# registrar will use when selecting phantoms.
 bidirectional_api_generation = 957