From 8c83de09e40d307babe962972c8cbe2c4bb1492d Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Sat, 30 Mar 2024 16:45:40 -0700
Subject: [PATCH 01/12] chore: Move Quic settings into alphabetical order

---
 .../src/settings/libp2p_config.rs             | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/homestar-runtime/src/settings/libp2p_config.rs b/homestar-runtime/src/settings/libp2p_config.rs
index 932fa343..694c768b 100644
--- a/homestar-runtime/src/settings/libp2p_config.rs
+++ b/homestar-runtime/src/settings/libp2p_config.rs
@@ -51,13 +51,6 @@ pub struct Libp2p {
     pub(crate) bootstrap_interval: Duration,
 }
 
-/// DHT settings.
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
-pub(crate) struct Quic {
-    /// Enable Quic transport.
-    pub(crate) enable: bool,
-}
-
 /// DHT settings.
 #[serde_as]
 #[derive(Builder, Clone, Debug, Serialize, Deserialize, PartialEq)]
@@ -127,6 +120,13 @@ pub struct Pubsub {
     pub(crate) mesh_outbound_min: usize,
 }
 
+/// Quic settings.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub(crate) struct Quic {
+    /// Enable Quic transport.
+    pub(crate) enable: bool,
+}
+
 /// Rendezvous settings.
 #[serde_as]
 #[derive(Builder, Clone, Debug, Serialize, Deserialize, PartialEq)]
@@ -193,12 +193,6 @@ impl Default for Dht {
     }
 }
 
-impl Default for Quic {
-    fn default() -> Self {
-        Self { enable: true }
-    }
-}
-
 impl Default for Mdns {
     fn default() -> Self {
         Self {
@@ -225,6 +219,12 @@ impl Default for Pubsub {
     }
 }
 
+impl Default for Quic {
+    fn default() -> Self {
+        Self { enable: true }
+    }
+}
+
 impl Default for Rendezvous {
     fn default() -> Self {
         Self {

From 5c966e94798279878731a383372b5102ba828e3f Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Mon, 1 Apr 2024 09:35:31 -0700
Subject: [PATCH 02/12] feat: Add AutoNAT behavior and config

---
 Cargo.lock                                    | 22 ++++++++++++++++++
 homestar-runtime/Cargo.toml                   |  1 +
 .../src/event_handler/swarm_event.rs          | 15 +++++++++++-
 homestar-runtime/src/lib.rs                   |  2 +-
 homestar-runtime/src/network/swarm.rs         | 18 +++++++++++++++
 homestar-runtime/src/settings.rs              |  2 +-
 .../src/settings/libp2p_config.rs             | 23 +++++++++++++++++++
 7 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d9e6c188..e43dada9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3690,6 +3690,7 @@ dependencies = [
  "getrandom",
  "instant",
  "libp2p-allow-block-list",
+ "libp2p-autonat",
  "libp2p-connection-limits",
  "libp2p-core",
  "libp2p-dns",
@@ -3726,6 +3727,27 @@ dependencies = [
  "void",
 ]
 
+[[package]]
+name = "libp2p-autonat"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d95151726170e41b591735bf95c42b888fe4aa14f65216a9fbf0edcc04510586"
+dependencies = [
+ "async-trait",
+ "asynchronous-codec 0.6.2",
+ "futures",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-request-response",
+ "libp2p-swarm",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.2.0",
+ "rand",
+ "tracing",
+]
+
 [[package]]
 name = "libp2p-connection-limits"
 version = "0.3.1"
diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml
index 204e3f7a..d8c45c0b 100644
--- a/homestar-runtime/Cargo.toml
+++ b/homestar-runtime/Cargo.toml
@@ -97,6 +97,7 @@ jsonrpsee = { version = "0.21", default-features = false, features = [
 ] }
 libipld = { workspace = true }
 libp2p = { version = "0.53", default-features = false, features = [
+  "autonat",
   "dns",
   "kad",
   "request-response",
diff --git a/homestar-runtime/src/event_handler/swarm_event.rs b/homestar-runtime/src/event_handler/swarm_event.rs
index cedcce13..424d19f9 100644
--- a/homestar-runtime/src/event_handler/swarm_event.rs
+++ b/homestar-runtime/src/event_handler/swarm_event.rs
@@ -27,7 +27,7 @@ use libipld::Cid;
 #[cfg(feature = "websocket-notify")]
 use libp2p::Multiaddr;
 use libp2p::{
-    gossipsub, identify, kad,
+    autonat, gossipsub, identify, kad,
     kad::{AddProviderOk, BootstrapOk, GetProvidersOk, GetRecordOk, PutRecordOk, QueryResult},
     mdns,
     multiaddr::Protocol,
@@ -106,6 +106,19 @@ async fn handle_swarm_event<DB: Database>(
     event_handler: &mut EventHandler<DB>,
 ) {
     match event {
+        SwarmEvent::Behaviour(ComposedEvent::Autonat(autonat_event)) => {
+            match autonat_event {
+                autonat::Event::InboundProbe { .. } => {
+                    // TODO
+                }
+                autonat::Event::OutboundProbe { .. } => {
+                    // TODO
+                }
+                autonat::Event::StatusChanged { .. } => {
+                    // TODO
+                }
+            }
+        }
         SwarmEvent::Behaviour(ComposedEvent::Identify(identify_event)) => {
             match identify_event {
                 identify::Event::Error { peer_id, error } => {
diff --git a/homestar-runtime/src/lib.rs b/homestar-runtime/src/lib.rs
index f23f0d6f..334764a7 100644
--- a/homestar-runtime/src/lib.rs
+++ b/homestar-runtime/src/lib.rs
@@ -84,7 +84,7 @@ pub(crate) use scheduler::TaskScheduler;
 #[cfg(feature = "ipfs")]
 pub use settings::IpfsBuilder;
 pub use settings::{
-    DatabaseBuilder, Dht, ExistingKeyPath, KeyType, Libp2p, Mdns, MetricsBuilder,
+    Autonat, DatabaseBuilder, Dht, ExistingKeyPath, KeyType, Libp2p, Mdns, MetricsBuilder,
     MonitoringBuilder, NetworkBuilder, NodeBuilder, PubkeyConfig, Pubsub, RNGSeed, Rendezvous,
     RpcBuilder, Settings, SettingsBuilder, WebserverBuilder,
 };
diff --git a/homestar-runtime/src/network/swarm.rs b/homestar-runtime/src/network/swarm.rs
index 13292f77..515c8f9d 100644
--- a/homestar-runtime/src/network/swarm.rs
+++ b/homestar-runtime/src/network/swarm.rs
@@ -13,6 +13,7 @@ use enum_assoc::Assoc;
 use faststr::FastStr;
 use futures::future::Either;
 use libp2p::{
+    autonat,
     core::{
         muxing::StreamMuxerBox,
         transport::{self, OptionalTransport},
@@ -62,6 +63,13 @@ pub(crate) async fn new(settings: &settings::Network) -> Result<Swarm<ComposedBe
     let mut swarm = Swarm::new(
         transport,
         ComposedBehaviour {
+            autonat: autonat::Behaviour::new(
+                keypair.clone().public().to_peer_id(),
+                autonat::Config {
+                    only_global_ips: settings.libp2p().autonat.only_global_ips,
+                    ..Default::default()
+                },
+            ),
             gossipsub: Toggle::from(if settings.libp2p.pubsub.enable {
                 Some(pubsub::new(keypair.clone(), settings.libp2p().pubsub())?)
             } else {
@@ -275,6 +283,8 @@ impl fmt::Display for CapsuleTag {
 /// Custom event types to listen for and respond to.
 #[derive(Debug)]
 pub(crate) enum ComposedEvent {
+    /// [autonat::Event] event.
+    Autonat(autonat::Event),
     /// [gossipsub::Event] event.
     Gossipsub(Box<gossipsub::Event>),
     /// [kad::Event] event.
@@ -303,6 +313,8 @@ pub(crate) enum TopicMessage {
 #[derive(NetworkBehaviour)]
 #[behaviour(to_swarm = "ComposedEvent")]
 pub(crate) struct ComposedBehaviour {
+    /// [autonat::Behaviour] behaviour.
+    pub(crate) autonat: autonat::Behaviour,
     /// [gossipsub::Behaviour] behaviour.
     pub(crate) gossipsub: Toggle<gossipsub::Behaviour>,
     /// In-memory [kademlia: kad::Behaviour] behaviour.
@@ -361,6 +373,12 @@ impl ComposedBehaviour {
     }
 }
 
+impl From<autonat::Event> for ComposedEvent {
+    fn from(event: autonat::Event) -> Self {
+        ComposedEvent::Autonat(event)
+    }
+}
+
 impl From<gossipsub::Event> for ComposedEvent {
     fn from(event: gossipsub::Event) -> Self {
         ComposedEvent::Gossipsub(Box::new(event))
diff --git a/homestar-runtime/src/settings.rs b/homestar-runtime/src/settings.rs
index 926d0aca..eb044c61 100644
--- a/homestar-runtime/src/settings.rs
+++ b/homestar-runtime/src/settings.rs
@@ -16,7 +16,7 @@ use std::{
 
 mod libp2p_config;
 mod pubkey_config;
-pub use libp2p_config::{Dht, Libp2p, Mdns, Pubsub, Rendezvous};
+pub use libp2p_config::{Autonat, Dht, Libp2p, Mdns, Pubsub, Rendezvous};
 pub use pubkey_config::{ExistingKeyPath, KeyType, PubkeyConfig, RNGSeed};
 
 #[cfg(target_os = "windows")]
diff --git a/homestar-runtime/src/settings/libp2p_config.rs b/homestar-runtime/src/settings/libp2p_config.rs
index 694c768b..204e0d8b 100644
--- a/homestar-runtime/src/settings/libp2p_config.rs
+++ b/homestar-runtime/src/settings/libp2p_config.rs
@@ -16,6 +16,8 @@ pub struct Libp2p {
     /// network.
     #[serde_as(as = "Vec<serde_with::DisplayFromStr>")]
     pub(crate) announce_addresses: Vec<libp2p::Multiaddr>,
+    /// Autonat DHT Settings
+    pub(crate) autonat: Autonat,
     /// Kademlia DHT Settings
     pub(crate) dht: Dht,
     #[serde_as(as = "DurationSeconds<u64>")]
@@ -51,6 +53,18 @@ pub struct Libp2p {
     pub(crate) bootstrap_interval: Duration,
 }
 
+/// Autonat settings.
+#[serde_as]
+#[derive(Builder, Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[builder(default)]
+#[serde(default)]
+pub struct Autonat {
+    /// Use global IP addresses only. A server will only fulfill probe requests
+    /// for public addresses, and a client will only request probes
+    /// from servers at public addresses.
+    pub(crate) only_global_ips: bool,
+}
+
 /// DHT settings.
 #[serde_as]
 #[derive(Builder, Clone, Debug, Serialize, Deserialize, PartialEq)]
@@ -149,6 +163,7 @@ impl Default for Libp2p {
     fn default() -> Self {
         Self {
             announce_addresses: Vec::new(),
+            autonat: Autonat::default(),
             dht: Dht::default(),
             // https://github.com/libp2p/rust-libp2p/pull/4967
             // https://github.com/libp2p/rust-libp2p/pull/4887
@@ -180,6 +195,14 @@ impl Libp2p {
     }
 }
 
+impl Default for Autonat {
+    fn default() -> Self {
+        Self {
+            only_global_ips: true,
+        }
+    }
+}
+
 impl Default for Dht {
     fn default() -> Self {
         Self {

From f037a7b63b4bebebbc7c891b7e4a138fcae572f8 Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Mon, 1 Apr 2024 14:23:13 -0700
Subject: [PATCH 03/12] chore: Add more settings

---
 homestar-runtime/src/network/swarm.rs         |  5 ++++-
 .../src/settings/libp2p_config.rs             | 21 ++++++++++++++++---
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/homestar-runtime/src/network/swarm.rs b/homestar-runtime/src/network/swarm.rs
index 515c8f9d..8c67e901 100644
--- a/homestar-runtime/src/network/swarm.rs
+++ b/homestar-runtime/src/network/swarm.rs
@@ -66,7 +66,10 @@ pub(crate) async fn new(settings: &settings::Network) -> Result<Swarm<ComposedBe
             autonat: autonat::Behaviour::new(
                 keypair.clone().public().to_peer_id(),
                 autonat::Config {
-                    only_global_ips: settings.libp2p().autonat.only_global_ips,
+                    boot_delay: settings.libp2p().autonat().boot_delay,
+                    retry_interval: settings.libp2p().autonat().retry_interval,
+                    throttle_server_period: settings.libp2p().autonat().throttle_server_period,
+                    only_global_ips: settings.libp2p().autonat().only_global_ips,
                     ..Default::default()
                 },
             ),
diff --git a/homestar-runtime/src/settings/libp2p_config.rs b/homestar-runtime/src/settings/libp2p_config.rs
index 204e0d8b..9b6d4c32 100644
--- a/homestar-runtime/src/settings/libp2p_config.rs
+++ b/homestar-runtime/src/settings/libp2p_config.rs
@@ -55,10 +55,17 @@ pub struct Libp2p {
 
 /// Autonat settings.
 #[serde_as]
-#[derive(Builder, Clone, Debug, Serialize, Deserialize, PartialEq)]
-#[builder(default)]
-#[serde(default)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct Autonat {
+    /// Initial delay before starting the fist probe.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub(crate) boot_delay: Duration,
+    /// Probe interval when max confidence has not been achieved
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub(crate) retry_interval: Duration,
+    /// Throttle period before re-using a peer as server for a dial-request.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub(crate) throttle_server_period: Duration,
     /// Use global IP addresses only. A server will only fulfill probe requests
     /// for public addresses, and a client will only request probes
     /// from servers at public addresses.
@@ -184,6 +191,11 @@ impl Default for Libp2p {
 }
 
 impl Libp2p {
+    /// Autonat settings getter.
+    pub(crate) fn autonat(&self) -> &Autonat {
+        &self.autonat
+    }
+
     /// DHT settings getter.
     pub(crate) fn dht(&self) -> &Dht {
         &self.dht
@@ -198,6 +210,9 @@ impl Libp2p {
 impl Default for Autonat {
     fn default() -> Self {
         Self {
+            boot_delay: Duration::from_secs(15),
+            retry_interval: Duration::from_secs(90),
+            throttle_server_period: Duration::from_secs(90),
             only_global_ips: true,
         }
     }

From 643b3c9929e70c6394409d7103ca3c6d22c556b4 Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Mon, 1 Apr 2024 16:46:15 -0700
Subject: [PATCH 04/12] chore: Add NatStatus to_tuple extension

---
 homestar-runtime/src/libp2p/mod.rs        |  1 +
 homestar-runtime/src/libp2p/nat_status.rs | 37 +++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 homestar-runtime/src/libp2p/nat_status.rs

diff --git a/homestar-runtime/src/libp2p/mod.rs b/homestar-runtime/src/libp2p/mod.rs
index 3adffaa4..1acdebe0 100644
--- a/homestar-runtime/src/libp2p/mod.rs
+++ b/homestar-runtime/src/libp2p/mod.rs
@@ -1,3 +1,4 @@
 //! libp2p utilities.
 
 pub(crate) mod multiaddr;
+pub(crate) mod nat_status;
diff --git a/homestar-runtime/src/libp2p/nat_status.rs b/homestar-runtime/src/libp2p/nat_status.rs
new file mode 100644
index 00000000..0e8bdbbf
--- /dev/null
+++ b/homestar-runtime/src/libp2p/nat_status.rs
@@ -0,0 +1,37 @@
+/// NatStatus extension methods.
+use libp2p::{autonat::NatStatus, Multiaddr};
+
+/// [NatStatus] extension trait.
+pub(crate) trait NatStatusExt {
+    fn to_tuple(&self) -> (String, Option<Multiaddr>);
+}
+
+impl NatStatusExt for NatStatus {
+    fn to_tuple(&self) -> (String, Option<Multiaddr>) {
+        match &self {
+            NatStatus::Public(address) => ("Public".to_string(), Some(address.to_owned())),
+            NatStatus::Private => ("Private".to_string(), None),
+            NatStatus::Unknown => ("Unknown".to_string(), None),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn converts_nat_status_to_tuple() {
+        let address: Multiaddr = "/ip4/127.0.0.1/tcp/7001".parse().unwrap();
+        let public_status = NatStatus::Public(address.clone());
+        let private_status = NatStatus::Private;
+        let unknown_status = NatStatus::Unknown;
+
+        assert_eq!(
+            public_status.to_tuple(),
+            ("Public".to_string(), Some(address))
+        );
+        assert_eq!(private_status.to_tuple(), ("Private".to_string(), None));
+        assert_eq!(unknown_status.to_tuple(), ("Unknown".to_string(), None));
+    }
+}

From 62fc71a99adfb5c07e8696c0cfa1cc7799f0713b Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Mon, 1 Apr 2024 16:49:57 -0700
Subject: [PATCH 05/12] chore: Add StatusChangedAutonat notification

---
 .../src/event_handler/notification.rs         |  4 +-
 .../src/event_handler/notification/network.rs | 41 ++++++++-
 .../notification/network/autonat.rs           | 89 +++++++++++++++++++
 .../src/event_handler/swarm_event.rs          | 27 ++++--
 4 files changed, 150 insertions(+), 11 deletions(-)
 create mode 100644 homestar-runtime/src/event_handler/notification/network/autonat.rs

diff --git a/homestar-runtime/src/event_handler/notification.rs b/homestar-runtime/src/event_handler/notification.rs
index 69611a0d..abe5867c 100644
--- a/homestar-runtime/src/event_handler/notification.rs
+++ b/homestar-runtime/src/event_handler/notification.rs
@@ -20,8 +20,8 @@ pub(crate) use network::{
     NetworkNotification, NewListenAddr, OutgoingConnectionError, PeerRegisteredRendezvous,
     PublishedReceiptPubsub, PutReceiptDht, PutWorkflowInfoDht, ReceiptQuorumFailureDht,
     ReceiptQuorumSuccessDht, ReceivedReceiptPubsub, ReceivedWorkflowInfo, RegisteredRendezvous,
-    SentWorkflowInfo, WorkflowInfoQuorumFailureDht, WorkflowInfoQuorumSuccessDht,
-    WorkflowInfoSource,
+    SentWorkflowInfo, StatusChangedAutonat, WorkflowInfoQuorumFailureDht,
+    WorkflowInfoQuorumSuccessDht, WorkflowInfoSource,
 };
 pub(crate) use receipt::ReceiptNotification;
 
diff --git a/homestar-runtime/src/event_handler/notification/network.rs b/homestar-runtime/src/event_handler/notification/network.rs
index 63e3eeae..1a932305 100644
--- a/homestar-runtime/src/event_handler/notification/network.rs
+++ b/homestar-runtime/src/event_handler/notification/network.rs
@@ -3,18 +3,19 @@
 //! [swarm]: libp2p::swarm::Swarm
 
 use anyhow::anyhow;
-
 use homestar_invocation::ipld::DagJson;
 use libipld::{serde::from_ipld, Ipld};
 use schemars::JsonSchema;
 use std::{collections::BTreeMap, fmt};
 
+pub(crate) mod autonat;
 pub(crate) mod connection;
 pub(crate) mod dht;
 pub(crate) mod mdns;
 pub(crate) mod pubsub;
 pub(crate) mod rendezvous;
 pub(crate) mod req_resp;
+pub(crate) use autonat::StatusChangedAutonat;
 pub(crate) use connection::{
     ConnectionClosed, ConnectionEstablished, IncomingConnectionError, NewListenAddr,
     OutgoingConnectionError,
@@ -43,12 +44,15 @@ pub enum NetworkNotification {
     /// Connection closed notification.
     #[schemars(rename = "connection_closed")]
     ConnnectionClosed(ConnectionClosed),
-    /// Outgoing conenction error notification.
+    /// Outgoing connection error notification.
     #[schemars(rename = "outgoing_connection_error")]
     OutgoingConnectionError(OutgoingConnectionError),
-    /// Incoming conenction error notification.
+    /// Incoming connection error notification.
     #[schemars(rename = "incoming_connection_error")]
     IncomingConnectionError(IncomingConnectionError),
+    /// Autonat status changed notification.
+    #[schemars(rename = "status_changed_autonat")]
+    StatusChangedAutonat(StatusChangedAutonat),
     /// mDNS discovered notification.
     #[schemars(rename = "discovered_mdns")]
     DiscoveredMdns(DiscoveredMdns),
@@ -120,6 +124,7 @@ impl fmt::Display for NetworkNotification {
             NetworkNotification::IncomingConnectionError(_) => {
                 write!(f, "incoming_connection_error")
             }
+            NetworkNotification::StatusChangedAutonat(_) => write!(f, "status_changed_autonat"),
             NetworkNotification::DiscoveredMdns(_) => write!(f, "discovered_mdns"),
             NetworkNotification::DiscoveredRendezvous(_) => write!(f, "discovered_rendezvous"),
             NetworkNotification::RegisteredRendezvous(_) => write!(f, "registered_rendezvous"),
@@ -180,6 +185,10 @@ impl From<NetworkNotification> for Ipld {
                 "incoming_connection_error".into(),
                 n.into(),
             )])),
+            NetworkNotification::StatusChangedAutonat(n) => Ipld::Map(BTreeMap::from([(
+                "status_changed_autonat".into(),
+                n.into(),
+            )])),
             NetworkNotification::DiscoveredMdns(n) => {
                 Ipld::Map(BTreeMap::from([("discovered_mdns".into(), n.into())]))
             }
@@ -267,6 +276,9 @@ impl TryFrom<Ipld> for NetworkNotification {
                 "incoming_connection_error" => Ok(NetworkNotification::IncomingConnectionError(
                     IncomingConnectionError::try_from(val.to_owned())?,
                 )),
+                "status_changed_autonat" => Ok(NetworkNotification::StatusChangedAutonat(
+                    StatusChangedAutonat::try_from(val.to_owned())?,
+                )),
                 "discovered_mdns" => Ok(NetworkNotification::DiscoveredMdns(
                     DiscoveredMdns::try_from(val.to_owned())?,
                 )),
@@ -333,10 +345,12 @@ impl TryFrom<Ipld> for NetworkNotification {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::libp2p::nat_status::NatStatusExt;
     use faststr::FastStr;
     use homestar_invocation::test_utils::cid::generate_cid;
     use libipld::Cid;
     use libp2p::{
+        autonat::NatStatus,
         swarm::{DialError, ListenError},
         Multiaddr, PeerId,
     };
@@ -350,6 +364,7 @@ mod test {
         cid: Cid,
         connected_peer_count: usize,
         name: FastStr,
+        nat_status: NatStatus,
         num_tasks: u32,
         peer_id: PeerId,
         peers: Vec<PeerId>,
@@ -371,6 +386,7 @@ mod test {
             cid: generate_cid(&mut thread_rng()),
             connected_peer_count: 1,
             name: FastStr::new("Strong Bad"),
+            nat_status: NatStatus::Public(Multiaddr::from_str("/ip4/127.0.0.1/tcp/7002").unwrap()),
             num_tasks: 1,
             peer_id: PeerId::random(),
             peers: vec![PeerId::random(), PeerId::random()],
@@ -411,6 +427,7 @@ mod test {
             cid,
             connected_peer_count,
             name,
+            nat_status,
             num_tasks,
             peer_id,
             peers,
@@ -428,6 +445,7 @@ mod test {
         let outgoing_connection_error =
             OutgoingConnectionError::new(Some(peer_id), DialError::NoAddresses);
         let incoming_connection_error = IncomingConnectionError::new(ListenError::Aborted);
+        let status_changed_autonat = StatusChangedAutonat::new(nat_status);
         let discovered_mdns = DiscoveredMdns::new(peers_map);
         let discovered_rendezvous = DiscoveredRendezvous::new(peer_id, peers_map_vec_addr);
         let registered_rendezvous = RegisteredRendezvous::new(peer_id);
@@ -506,6 +524,10 @@ mod test {
                 incoming_connection_error.timestamp().to_owned(),
                 NetworkNotification::IncomingConnectionError(incoming_connection_error),
             ),
+            (
+                status_changed_autonat.timestamp().to_owned(),
+                NetworkNotification::StatusChangedAutonat(status_changed_autonat),
+            ),
             (
                 discovered_mdns.timestamp().to_owned(),
                 NetworkNotification::DiscoveredMdns(discovered_mdns),
@@ -584,6 +606,7 @@ mod test {
             cid,
             connected_peer_count,
             name,
+            nat_status,
             num_tasks,
             peer_id,
             peers,
@@ -623,6 +646,18 @@ mod test {
                 assert_eq!(n.timestamp(), timestamp);
                 assert_eq!(n.error().to_string(), ListenError::Aborted.to_string());
             }
+            NetworkNotification::StatusChangedAutonat(n) => {
+                let (status, address) = nat_status.to_tuple();
+
+                assert_eq!(n.timestamp(), timestamp);
+                assert_eq!(n.status(), &status);
+                assert_eq!(
+                    n.address()
+                        .as_ref()
+                        .map(|a| Multiaddr::from_str(&a).unwrap()),
+                    address
+                );
+            }
             NetworkNotification::DiscoveredMdns(n) => {
                 assert_eq!(n.timestamp(), timestamp);
 
diff --git a/homestar-runtime/src/event_handler/notification/network/autonat.rs b/homestar-runtime/src/event_handler/notification/network/autonat.rs
new file mode 100644
index 00000000..a037c853
--- /dev/null
+++ b/homestar-runtime/src/event_handler/notification/network/autonat.rs
@@ -0,0 +1,89 @@
+//! Notification types for [swarm] autonat events.
+//!
+//! [swarm]: libp2p::swarm::Swarm
+
+use crate::libp2p::nat_status::NatStatusExt;
+use anyhow::anyhow;
+use chrono::prelude::Utc;
+use derive_getters::Getters;
+use homestar_invocation::ipld::DagJson;
+use libipld::{serde::from_ipld, Ipld};
+use libp2p::autonat::NatStatus;
+use schemars::JsonSchema;
+use std::collections::BTreeMap;
+
+const ADDRESS_KEY: &str = "address";
+const STATUS_KEY: &str = "status";
+const TIMESTAMP_KEY: &str = "timestamp";
+
+#[derive(Debug, Clone, Getters, JsonSchema)]
+#[schemars(rename = "status_changed_autonat")]
+pub struct StatusChangedAutonat {
+    timestamp: i64,
+    status: String,
+    address: Option<String>,
+}
+
+impl StatusChangedAutonat {
+    pub(crate) fn new(status: NatStatus) -> StatusChangedAutonat {
+        let (status, address) = status.to_tuple();
+
+        StatusChangedAutonat {
+            timestamp: Utc::now().timestamp_millis(),
+            status: status.to_string(),
+            address: address.map(|a| a.to_string()),
+        }
+    }
+}
+
+impl DagJson for StatusChangedAutonat {}
+
+impl From<StatusChangedAutonat> for Ipld {
+    fn from(notification: StatusChangedAutonat) -> Self {
+        Ipld::Map(BTreeMap::from([
+            (TIMESTAMP_KEY.into(), notification.timestamp.into()),
+            (STATUS_KEY.into(), notification.status.into()),
+            (
+                ADDRESS_KEY.into(),
+                notification
+                    .address
+                    .map(|peer_id| peer_id.into())
+                    .unwrap_or(Ipld::Null),
+            ),
+        ]))
+    }
+}
+
+impl TryFrom<Ipld> for StatusChangedAutonat {
+    type Error = anyhow::Error;
+
+    fn try_from(ipld: Ipld) -> Result<Self, Self::Error> {
+        let map = from_ipld::<BTreeMap<String, Ipld>>(ipld)?;
+
+        let timestamp = from_ipld(
+            map.get(TIMESTAMP_KEY)
+                .ok_or_else(|| anyhow!("missing {TIMESTAMP_KEY}"))?
+                .to_owned(),
+        )?;
+
+        let status = from_ipld(
+            map.get(STATUS_KEY)
+                .ok_or_else(|| anyhow!("missing {STATUS_KEY}"))?
+                .to_owned(),
+        )?;
+
+        let address = map
+            .get(ADDRESS_KEY)
+            .and_then(|ipld| match ipld {
+                Ipld::Null => None,
+                ipld => Some(ipld),
+            })
+            .and_then(|ipld| from_ipld(ipld.to_owned()).ok());
+
+        Ok(StatusChangedAutonat {
+            timestamp,
+            status,
+            address,
+        })
+    }
+}
diff --git a/homestar-runtime/src/event_handler/swarm_event.rs b/homestar-runtime/src/event_handler/swarm_event.rs
index 424d19f9..310b9129 100644
--- a/homestar-runtime/src/event_handler/swarm_event.rs
+++ b/homestar-runtime/src/event_handler/swarm_event.rs
@@ -108,14 +108,29 @@ async fn handle_swarm_event<DB: Database>(
     match event {
         SwarmEvent::Behaviour(ComposedEvent::Autonat(autonat_event)) => {
             match autonat_event {
-                autonat::Event::InboundProbe { .. } => {
-                    // TODO
+                autonat::Event::InboundProbe(event) => {
+                    // TODO Add log
+                    println!("INBOUND PROBE EVENT: {event:?}")
                 }
-                autonat::Event::OutboundProbe { .. } => {
-                    // TODO
+                autonat::Event::OutboundProbe(event) => {
+                    // TODO Add log
+                    println!("OUTBOUND PROBE EVENT: {event:?}");
+                    println!(
+                        "CONFIDENCE: {}",
+                        event_handler.swarm.behaviour().autonat.confidence()
+                    )
                 }
-                autonat::Event::StatusChanged { .. } => {
-                    // TODO
+                autonat::Event::StatusChanged { old, new } => {
+                    // TODO Add log
+                    println!("STATUS CHANGED: Old - {old:?}, New - {new:?}");
+
+                    #[cfg(feature = "websocket-notify")]
+                    notification::emit_network_event(
+                        event_handler.ws_evt_sender(),
+                        NetworkNotification::StatusChangedAutonat(
+                            notification::StatusChangedAutonat::new(new),
+                        ),
+                    );
                 }
             }
         }

From 871ba5300e3afcb2696e61a31f9b5cb84eff41bc Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Tue, 2 Apr 2024 11:58:12 -0700
Subject: [PATCH 06/12] chore: Add and remove external addresses by NAT status

---
 .../src/event_handler/swarm_event.rs          | 32 +++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/homestar-runtime/src/event_handler/swarm_event.rs b/homestar-runtime/src/event_handler/swarm_event.rs
index 310b9129..fdcef0cd 100644
--- a/homestar-runtime/src/event_handler/swarm_event.rs
+++ b/homestar-runtime/src/event_handler/swarm_event.rs
@@ -27,7 +27,8 @@ use libipld::Cid;
 #[cfg(feature = "websocket-notify")]
 use libp2p::Multiaddr;
 use libp2p::{
-    autonat, gossipsub, identify, kad,
+    autonat::{self, NatStatus},
+    gossipsub, identify, kad,
     kad::{AddProviderOk, BootstrapOk, GetProvidersOk, GetRecordOk, PutRecordOk, QueryResult},
     mdns,
     multiaddr::Protocol,
@@ -121,8 +122,33 @@ async fn handle_swarm_event<DB: Database>(
                     )
                 }
                 autonat::Event::StatusChanged { old, new } => {
-                    // TODO Add log
-                    println!("STATUS CHANGED: Old - {old:?}, New - {new:?}");
+                    match &new {
+                        NatStatus::Public(address) => {
+                            event_handler.swarm.add_external_address(address.clone());
+
+                            info!(
+                                subject = "libp2p.autonat.status_change",
+                                category = "handle_swarm_event",
+                                address = address.to_string(),
+                                "Confirmed a public address",
+                            );
+                        }
+                        _ => {
+                            if let NatStatus::Public(address) = old {
+                                // Announce addresses are configured and should not be removed
+                                if !event_handler.announce_addresses.contains(&address) {
+                                    event_handler.swarm.remove_external_address(&address);
+
+                                    info!(
+                                        subject = "libp2p.autonat.status_change",
+                                        category = "handle_swarm_event",
+                                        address = address.to_string(),
+                                        "Removed an address that is no longer public",
+                                    );
+                                }
+                            }
+                        }
+                    }
 
                     #[cfg(feature = "websocket-notify")]
                     notification::emit_network_event(

From a0eb7c6f45d605edb93ad46b1cadcf99c3e93e77 Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Tue, 2 Apr 2024 14:19:59 -0700
Subject: [PATCH 07/12] chore: Add inbound and outbound probe debug logs

---
 .../src/event_handler/swarm_event.rs          | 75 +++++++++++++++----
 1 file changed, 61 insertions(+), 14 deletions(-)

diff --git a/homestar-runtime/src/event_handler/swarm_event.rs b/homestar-runtime/src/event_handler/swarm_event.rs
index fdcef0cd..a6f879f4 100644
--- a/homestar-runtime/src/event_handler/swarm_event.rs
+++ b/homestar-runtime/src/event_handler/swarm_event.rs
@@ -109,18 +109,65 @@ async fn handle_swarm_event<DB: Database>(
     match event {
         SwarmEvent::Behaviour(ComposedEvent::Autonat(autonat_event)) => {
             match autonat_event {
-                autonat::Event::InboundProbe(event) => {
-                    // TODO Add log
-                    println!("INBOUND PROBE EVENT: {event:?}")
-                }
-                autonat::Event::OutboundProbe(event) => {
-                    // TODO Add log
-                    println!("OUTBOUND PROBE EVENT: {event:?}");
-                    println!(
-                        "CONFIDENCE: {}",
-                        event_handler.swarm.behaviour().autonat.confidence()
-                    )
-                }
+                autonat::Event::InboundProbe(event) => match event {
+                    autonat::InboundProbeEvent::Request {
+                        peer, addresses, ..
+                    } => {
+                        debug!(
+                            subject = "libp2p.autonat.inbound_probe",
+                            category = "handle_swarm_event",
+                            peer_id = peer.to_string(),
+                            addresses = ?addresses,
+                            "received a probe request",
+                        );
+                    }
+                    autonat::InboundProbeEvent::Response { peer, address, .. } => {
+                        debug!(
+                            subject = "libp2p.autonat.inbound_probe",
+                            category = "handle_swarm_event",
+                            peer_id = peer.to_string(),
+                            address = address.to_string(),
+                            "successfully probed an external address for a peer",
+                        );
+                    }
+                    autonat::InboundProbeEvent::Error { peer, error, .. } => {
+                        debug!(
+                            subject = "libp2p.autonat.inbound_probe",
+                            category = "handle_swarm_event",
+                            peer_id = peer.to_string(),
+                            error = ?error,
+                            "unable to probe a peer",
+                        );
+                    }
+                },
+                autonat::Event::OutboundProbe(event) => match event {
+                    autonat::OutboundProbeEvent::Request { peer, .. } => {
+                        debug!(
+                            subject = "libp2p.autonat.outbound_probe",
+                            category = "handle_swarm_event",
+                            peer_id = peer.to_string(),
+                            "requested a probe from a peer",
+                        );
+                    }
+                    autonat::OutboundProbeEvent::Response { peer, address, .. } => {
+                        debug!(
+                            subject = "libp2p.autonat.outbound_probe",
+                            category = "handle_swarm_event",
+                            peer_id = peer.to_string(),
+                            address = address.to_string(),
+                            "peer successfully probed an external address",
+                        );
+                    }
+                    autonat::OutboundProbeEvent::Error { peer, error, .. } => {
+                        debug!(
+                            subject = "libp2p.autonat.outbound_probe",
+                            category = "handle_swarm_event",
+                            peer_id = peer.map(|p| p.to_string()).unwrap_or("<none>".to_string()),
+                            error = ?error,
+                            "requested probe failed",
+                        );
+                    }
+                },
                 autonat::Event::StatusChanged { old, new } => {
                     match &new {
                         NatStatus::Public(address) => {
@@ -130,7 +177,7 @@ async fn handle_swarm_event<DB: Database>(
                                 subject = "libp2p.autonat.status_change",
                                 category = "handle_swarm_event",
                                 address = address.to_string(),
-                                "Confirmed a public address",
+                                "confirmed a public address",
                             );
                         }
                         _ => {
@@ -143,7 +190,7 @@ async fn handle_swarm_event<DB: Database>(
                                         subject = "libp2p.autonat.status_change",
                                         category = "handle_swarm_event",
                                         address = address.to_string(),
-                                        "Removed an address that is no longer public",
+                                        "removed an address that is no longer public",
                                     );
                                 }
                             }

From b897bae113b6b65dd0c521e2dd75de1801eb7a95 Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Tue, 2 Apr 2024 14:54:21 -0700
Subject: [PATCH 08/12] test: Add test_autonat_confirms_address_integration
 test

---
 homestar-runtime/tests/network.rs         |   2 +
 homestar-runtime/tests/network/autonat.rs | 175 ++++++++++++++++++++++
 2 files changed, 177 insertions(+)
 create mode 100644 homestar-runtime/tests/network/autonat.rs

diff --git a/homestar-runtime/tests/network.rs b/homestar-runtime/tests/network.rs
index 54aaef7c..1a0a6f4a 100644
--- a/homestar-runtime/tests/network.rs
+++ b/homestar-runtime/tests/network.rs
@@ -13,6 +13,8 @@ use std::{
     process::{Command, Stdio},
 };
 
+#[cfg(feature = "websocket-notify")]
+mod autonat;
 #[cfg(feature = "websocket-notify")]
 mod connection;
 #[cfg(all(feature = "websocket-notify", feature = "test-utils"))]
diff --git a/homestar-runtime/tests/network/autonat.rs b/homestar-runtime/tests/network/autonat.rs
new file mode 100644
index 00000000..bab4814d
--- /dev/null
+++ b/homestar-runtime/tests/network/autonat.rs
@@ -0,0 +1,175 @@
+use crate::{
+    make_config,
+    utils::{
+        check_for_line_with, kill_homestar, listen_addr, multiaddr, retrieve_output,
+        subscribe_network_events, wait_for_socket_connection, ChildGuard, ProcInfo,
+        TimeoutFutureExt, BIN_NAME, ED25519MULTIHASH, SECP256K1MULTIHASH,
+    },
+};
+use anyhow::Result;
+use once_cell::sync::Lazy;
+use std::{
+    path::PathBuf,
+    process::{Command, Stdio},
+    time::Duration,
+};
+
+static BIN: Lazy<PathBuf> = Lazy::new(|| assert_cmd::cargo::cargo_bin(BIN_NAME));
+
+#[test]
+#[serial_test::parallel]
+fn test_autonat_confirms_address_integration() -> Result<()> {
+    let proc_info1 = ProcInfo::new().unwrap();
+    let proc_info2 = ProcInfo::new().unwrap();
+
+    let rpc_port1 = proc_info1.rpc_port;
+    let rpc_port2 = proc_info2.rpc_port;
+    let metrics_port1 = proc_info1.metrics_port;
+    let metrics_port2 = proc_info2.metrics_port;
+    let ws_port1 = proc_info1.ws_port;
+    let ws_port2 = proc_info2.ws_port;
+    let listen_addr1 = listen_addr(proc_info1.listen_port);
+    let listen_addr2 = listen_addr(proc_info2.listen_port);
+    let node_addra = multiaddr(proc_info1.listen_port, ED25519MULTIHASH);
+
+    let toml = format!(
+        r#"
+        [node]
+        [node.network.keypair_config]
+        existing = {{ key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }}
+        [node.network.libp2p]
+        listen_address = "{listen_addr1}"
+        [node.network.libp2p.autonat]
+        boot_delay = 1
+        retry_interval = 3
+        throttle_server_period = 2
+        only_global_ips = false
+        [node.network.libp2p.mdns]
+        enable = false
+        [node.network.libp2p.rendezvous]
+        enable_client = false
+        [node.network.metrics]
+        port = {metrics_port1}
+        [node.network.rpc]
+        port = {rpc_port1}
+        [node.network.webserver]
+        port = {ws_port1}
+        "#
+    );
+    let config1 = make_config!(toml);
+
+    let homestar_proc1 = Command::new(BIN.as_os_str())
+        .env("RUST_BACKTRACE", "0")
+        .env(
+            "RUST_LOG",
+            "homestar=debug,homestar_runtime=debug,libp2p=debug,libp2p_gossipsub::behaviour=debug",
+        )
+        .arg("start")
+        .arg("-c")
+        .arg(config1.filename())
+        .arg("--db")
+        .arg(&proc_info1.db_path)
+        .stdout(Stdio::piped())
+        .spawn()
+        .unwrap();
+    let proc_guard1 = ChildGuard::new(homestar_proc1);
+
+    if wait_for_socket_connection(ws_port1, 1000).is_err() {
+        panic!("Homestar server/runtime failed to start in time");
+    }
+
+    tokio_test::block_on(async {
+        let toml2 = format!(
+            r#"
+            [node]
+            [node.network.keypair_config]
+            existing = {{ key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" }}
+            [node.network.libp2p]
+            listen_address = "{listen_addr2}"
+            node_addresses = ["{node_addra}"]
+            [node.network.libp2p.autonat]
+            boot_delay = 1
+            retry_interval = 3
+            throttle_server_period = 2
+            only_global_ips = false
+            [node.network.libp2p.mdns]
+            enable = false
+            [node.network.metrics]
+            port = {metrics_port2}
+            [node.network.libp2p.rendezvous]
+            enable_client = false
+            [node.network.rpc]
+            port = {rpc_port2}
+            [node.network.webserver]
+            port = {ws_port2}
+            "#
+        );
+        let config2 = make_config!(toml2);
+
+        let homestar_proc2 = Command::new(BIN.as_os_str())
+                .env("RUST_BACKTRACE", "0")
+                .env(
+                    "RUST_LOG",
+                    "homestar=debug,homestar_runtime=debug,libp2p=debug,libp2p_gossipsub::behaviour=debug",
+                )
+                .arg("start")
+                .arg("-c")
+                .arg(config2.filename())
+                .arg("--db")
+                .arg(&proc_info2.db_path)
+                .stdout(Stdio::piped())
+                .spawn()
+                .unwrap();
+        let proc_guard2 = ChildGuard::new(homestar_proc2);
+
+        let mut net_events = subscribe_network_events(ws_port1).await;
+        let sub = net_events.sub();
+
+        // Poll for status changed autonat message
+        loop {
+            if let Ok(msg) = sub.next().with_timeout(Duration::from_secs(30)).await {
+                let json: serde_json::Value =
+                    serde_json::from_slice(&msg.unwrap().unwrap()).unwrap();
+
+                if json["status_changed_autonat"].is_object()
+                    && json["status_changed_autonat"]["status"] == "Public"
+                {
+                    break;
+                }
+            } else {
+                panic!("Node two did not receive a NAT public status message in time.")
+            }
+        }
+
+        // Kill proceses.
+        let dead_proc1 = kill_homestar(proc_guard1.take(), None);
+        let dead_proc2 = kill_homestar(proc_guard2.take(), None);
+
+        // Retrieve logs.
+        let stdout1 = retrieve_output(dead_proc1);
+        let stdout2 = retrieve_output(dead_proc2);
+
+        // Check node one successfully probed an address for node two
+        let one_confirmed_address = check_for_line_with(
+            stdout1,
+            vec![
+                "successfully probed an external address for a peer",
+                SECP256K1MULTIHASH,
+            ],
+        );
+
+        // Check node two received a probe confirmation from node one
+        let two_received_address_confirmation = check_for_line_with(
+            stdout2,
+            vec![
+                "peer successfully probed an external address",
+                ED25519MULTIHASH,
+            ],
+        );
+
+        assert!(one_confirmed_address);
+        assert!(two_received_address_confirmation);
+    });
+
+    Ok(())
+}

From d87b73f318892e3a87daf4a2c8cd42f2dcac8210 Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Tue, 2 Apr 2024 15:14:18 -0700
Subject: [PATCH 09/12] chore: Rename only_global_ips setting to
 only_public_ips

---
 homestar-runtime/src/network/swarm.rs          | 2 +-
 homestar-runtime/src/settings/libp2p_config.rs | 6 +++---
 homestar-runtime/tests/network/autonat.rs      | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/homestar-runtime/src/network/swarm.rs b/homestar-runtime/src/network/swarm.rs
index 8c67e901..564a81ee 100644
--- a/homestar-runtime/src/network/swarm.rs
+++ b/homestar-runtime/src/network/swarm.rs
@@ -69,7 +69,7 @@ pub(crate) async fn new(settings: &settings::Network) -> Result<Swarm<ComposedBe
                     boot_delay: settings.libp2p().autonat().boot_delay,
                     retry_interval: settings.libp2p().autonat().retry_interval,
                     throttle_server_period: settings.libp2p().autonat().throttle_server_period,
-                    only_global_ips: settings.libp2p().autonat().only_global_ips,
+                    only_global_ips: settings.libp2p().autonat().only_public_ips,
                     ..Default::default()
                 },
             ),
diff --git a/homestar-runtime/src/settings/libp2p_config.rs b/homestar-runtime/src/settings/libp2p_config.rs
index 9b6d4c32..230d00b0 100644
--- a/homestar-runtime/src/settings/libp2p_config.rs
+++ b/homestar-runtime/src/settings/libp2p_config.rs
@@ -66,10 +66,10 @@ pub struct Autonat {
     /// Throttle period before re-using a peer as server for a dial-request.
     #[serde_as(as = "DurationSeconds<u64>")]
     pub(crate) throttle_server_period: Duration,
-    /// Use global IP addresses only. A server will only fulfill probe requests
+    /// Use public IP addresses only. A server will only fulfill probe requests
     /// for public addresses, and a client will only request probes
     /// from servers at public addresses.
-    pub(crate) only_global_ips: bool,
+    pub(crate) only_public_ips: bool,
 }
 
 /// DHT settings.
@@ -213,7 +213,7 @@ impl Default for Autonat {
             boot_delay: Duration::from_secs(15),
             retry_interval: Duration::from_secs(90),
             throttle_server_period: Duration::from_secs(90),
-            only_global_ips: true,
+            only_public_ips: true,
         }
     }
 }
diff --git a/homestar-runtime/tests/network/autonat.rs b/homestar-runtime/tests/network/autonat.rs
index bab4814d..fc7edbbf 100644
--- a/homestar-runtime/tests/network/autonat.rs
+++ b/homestar-runtime/tests/network/autonat.rs
@@ -43,7 +43,7 @@ fn test_autonat_confirms_address_integration() -> Result<()> {
         boot_delay = 1
         retry_interval = 3
         throttle_server_period = 2
-        only_global_ips = false
+        only_public_ips = false
         [node.network.libp2p.mdns]
         enable = false
         [node.network.libp2p.rendezvous]
@@ -91,7 +91,7 @@ fn test_autonat_confirms_address_integration() -> Result<()> {
             boot_delay = 1
             retry_interval = 3
             throttle_server_period = 2
-            only_global_ips = false
+            only_public_ips = false
             [node.network.libp2p.mdns]
             enable = false
             [node.network.metrics]

From 9843d70599b4ee1b88dc1a4ea8f1ada3b276d780 Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Tue, 2 Apr 2024 15:58:59 -0700
Subject: [PATCH 10/12] test: Add test_autonat_fails_private_ips_integration
 test

---
 homestar-runtime/tests/network/autonat.rs | 166 +++++++++++++++++++++-
 1 file changed, 165 insertions(+), 1 deletion(-)

diff --git a/homestar-runtime/tests/network/autonat.rs b/homestar-runtime/tests/network/autonat.rs
index fc7edbbf..7b913f5b 100644
--- a/homestar-runtime/tests/network/autonat.rs
+++ b/homestar-runtime/tests/network/autonat.rs
@@ -122,7 +122,11 @@ fn test_autonat_confirms_address_integration() -> Result<()> {
                 .unwrap();
         let proc_guard2 = ChildGuard::new(homestar_proc2);
 
-        let mut net_events = subscribe_network_events(ws_port1).await;
+        if wait_for_socket_connection(ws_port2, 1000).is_err() {
+            panic!("Homestar server/runtime failed to start in time");
+        }
+
+        let mut net_events = subscribe_network_events(ws_port2).await;
         let sub = net_events.sub();
 
         // Poll for status changed autonat message
@@ -173,3 +177,163 @@ fn test_autonat_confirms_address_integration() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+#[serial_test::parallel]
+fn test_autonat_fails_private_ips_integration() -> Result<()> {
+    let proc_info1 = ProcInfo::new().unwrap();
+    let proc_info2 = ProcInfo::new().unwrap();
+
+    let rpc_port1 = proc_info1.rpc_port;
+    let rpc_port2 = proc_info2.rpc_port;
+    let metrics_port1 = proc_info1.metrics_port;
+    let metrics_port2 = proc_info2.metrics_port;
+    let ws_port1 = proc_info1.ws_port;
+    let ws_port2 = proc_info2.ws_port;
+    let listen_addr1 = listen_addr(proc_info1.listen_port);
+    let listen_addr2 = listen_addr(proc_info2.listen_port);
+    let node_addra = multiaddr(proc_info1.listen_port, ED25519MULTIHASH);
+
+    // Node one will not accept probe requests from private IPs
+    let toml = format!(
+        r#"
+        [node]
+        [node.network.keypair_config]
+        existing = {{ key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }}
+        [node.network.libp2p]
+        listen_address = "{listen_addr1}"
+        [node.network.libp2p.autonat]
+        boot_delay = 1
+        retry_interval = 3
+        throttle_server_period = 2
+        only_public_ips = true
+        [node.network.libp2p.mdns]
+        enable = false
+        [node.network.libp2p.rendezvous]
+        enable_client = false
+        [node.network.metrics]
+        port = {metrics_port1}
+        [node.network.rpc]
+        port = {rpc_port1}
+        [node.network.webserver]
+        port = {ws_port1}
+        "#
+    );
+    let config1 = make_config!(toml);
+
+    let homestar_proc1 = Command::new(BIN.as_os_str())
+        .env("RUST_BACKTRACE", "0")
+        .env(
+            "RUST_LOG",
+            "homestar=debug,homestar_runtime=debug,libp2p=debug,libp2p_gossipsub::behaviour=debug",
+        )
+        .arg("start")
+        .arg("-c")
+        .arg(config1.filename())
+        .arg("--db")
+        .arg(&proc_info1.db_path)
+        .stdout(Stdio::piped())
+        .spawn()
+        .unwrap();
+    let proc_guard1 = ChildGuard::new(homestar_proc1);
+
+    if wait_for_socket_connection(ws_port1, 1000).is_err() {
+        panic!("Homestar server/runtime failed to start in time");
+    }
+
+    tokio_test::block_on(async {
+        // Node two will request probes from private IPs, but node one will refuse them
+        let toml2 = format!(
+            r#"
+            [node]
+            [node.network.keypair_config]
+            existing = {{ key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" }}
+            [node.network.libp2p]
+            listen_address = "{listen_addr2}"
+            node_addresses = ["{node_addra}"]
+            [node.network.libp2p.autonat]
+            boot_delay = 1
+            retry_interval = 3
+            throttle_server_period = 2
+            only_public_ips = false
+            [node.network.libp2p.mdns]
+            enable = false
+            [node.network.metrics]
+            port = {metrics_port2}
+            [node.network.libp2p.rendezvous]
+            enable_client = false
+            [node.network.rpc]
+            port = {rpc_port2}
+            [node.network.webserver]
+            port = {ws_port2}
+            "#
+        );
+        let config2 = make_config!(toml2);
+
+        let homestar_proc2 = Command::new(BIN.as_os_str())
+        .env("RUST_BACKTRACE", "0")
+        .env(
+            "RUST_LOG",
+            "homestar=debug,homestar_runtime=debug,libp2p=debug,libp2p_gossipsub::behaviour=debug",
+        )
+        .arg("start")
+        .arg("-c")
+        .arg(config2.filename())
+        .arg("--db")
+        .arg(&proc_info2.db_path)
+        .stdout(Stdio::piped())
+        .spawn()
+        .unwrap();
+        let proc_guard2 = ChildGuard::new(homestar_proc2);
+
+        let mut net_events = subscribe_network_events(ws_port1).await;
+        let sub = net_events.sub();
+
+        // Poll for connection established message
+        loop {
+            if let Ok(msg) = sub.next().with_timeout(Duration::from_secs(30)).await {
+                let json: serde_json::Value =
+                    serde_json::from_slice(&msg.unwrap().unwrap()).unwrap();
+
+                if json["connection_established"].is_object() {
+                    break;
+                }
+            } else {
+                panic!("Node one did not receive a connection established message in time.")
+            }
+        }
+
+        // Kill proceses.
+        let dead_proc1 = kill_homestar(proc_guard1.take(), Some(Duration::from_secs(2)));
+        let dead_proc2 = kill_homestar(proc_guard2.take(), Some(Duration::from_secs(2)));
+
+        // Retrieve logs.
+        let stdout1 = retrieve_output(dead_proc1);
+        let stdout2 = retrieve_output(dead_proc2);
+
+        // Check node one refused probe request from node two
+        let one_refused_probe_request = check_for_line_with(
+            stdout1,
+            vec![
+                "unable to probe a peer",
+                "Response(DialRefused)",
+                SECP256K1MULTIHASH,
+            ],
+        );
+
+        // Check node two received refusal from node one
+        let two_probe_request_failed = check_for_line_with(
+            stdout2,
+            vec![
+                "requested probe failed",
+                "Response(DialRefused)",
+                ED25519MULTIHASH,
+            ],
+        );
+
+        assert!(one_refused_probe_request);
+        assert!(two_probe_request_failed);
+    });
+
+    Ok(())
+}

From 3ace81e290ce299ca4d777f55622b271e91c6d1b Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Wed, 3 Apr 2024 10:03:51 -0700
Subject: [PATCH 11/12] refactor: Remove
 test_autonat_fails_private_ips_integration test

It was mostly testing the libp2p implementation, not our use of it.
---
 homestar-runtime/tests/network/autonat.rs | 160 ----------------------
 1 file changed, 160 deletions(-)

diff --git a/homestar-runtime/tests/network/autonat.rs b/homestar-runtime/tests/network/autonat.rs
index 7b913f5b..ec2b9801 100644
--- a/homestar-runtime/tests/network/autonat.rs
+++ b/homestar-runtime/tests/network/autonat.rs
@@ -177,163 +177,3 @@ fn test_autonat_confirms_address_integration() -> Result<()> {
 
     Ok(())
 }
-
-#[test]
-#[serial_test::parallel]
-fn test_autonat_fails_private_ips_integration() -> Result<()> {
-    let proc_info1 = ProcInfo::new().unwrap();
-    let proc_info2 = ProcInfo::new().unwrap();
-
-    let rpc_port1 = proc_info1.rpc_port;
-    let rpc_port2 = proc_info2.rpc_port;
-    let metrics_port1 = proc_info1.metrics_port;
-    let metrics_port2 = proc_info2.metrics_port;
-    let ws_port1 = proc_info1.ws_port;
-    let ws_port2 = proc_info2.ws_port;
-    let listen_addr1 = listen_addr(proc_info1.listen_port);
-    let listen_addr2 = listen_addr(proc_info2.listen_port);
-    let node_addra = multiaddr(proc_info1.listen_port, ED25519MULTIHASH);
-
-    // Node one will not accept probe requests from private IPs
-    let toml = format!(
-        r#"
-        [node]
-        [node.network.keypair_config]
-        existing = {{ key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }}
-        [node.network.libp2p]
-        listen_address = "{listen_addr1}"
-        [node.network.libp2p.autonat]
-        boot_delay = 1
-        retry_interval = 3
-        throttle_server_period = 2
-        only_public_ips = true
-        [node.network.libp2p.mdns]
-        enable = false
-        [node.network.libp2p.rendezvous]
-        enable_client = false
-        [node.network.metrics]
-        port = {metrics_port1}
-        [node.network.rpc]
-        port = {rpc_port1}
-        [node.network.webserver]
-        port = {ws_port1}
-        "#
-    );
-    let config1 = make_config!(toml);
-
-    let homestar_proc1 = Command::new(BIN.as_os_str())
-        .env("RUST_BACKTRACE", "0")
-        .env(
-            "RUST_LOG",
-            "homestar=debug,homestar_runtime=debug,libp2p=debug,libp2p_gossipsub::behaviour=debug",
-        )
-        .arg("start")
-        .arg("-c")
-        .arg(config1.filename())
-        .arg("--db")
-        .arg(&proc_info1.db_path)
-        .stdout(Stdio::piped())
-        .spawn()
-        .unwrap();
-    let proc_guard1 = ChildGuard::new(homestar_proc1);
-
-    if wait_for_socket_connection(ws_port1, 1000).is_err() {
-        panic!("Homestar server/runtime failed to start in time");
-    }
-
-    tokio_test::block_on(async {
-        // Node two will request probes from private IPs, but node one will refuse them
-        let toml2 = format!(
-            r#"
-            [node]
-            [node.network.keypair_config]
-            existing = {{ key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" }}
-            [node.network.libp2p]
-            listen_address = "{listen_addr2}"
-            node_addresses = ["{node_addra}"]
-            [node.network.libp2p.autonat]
-            boot_delay = 1
-            retry_interval = 3
-            throttle_server_period = 2
-            only_public_ips = false
-            [node.network.libp2p.mdns]
-            enable = false
-            [node.network.metrics]
-            port = {metrics_port2}
-            [node.network.libp2p.rendezvous]
-            enable_client = false
-            [node.network.rpc]
-            port = {rpc_port2}
-            [node.network.webserver]
-            port = {ws_port2}
-            "#
-        );
-        let config2 = make_config!(toml2);
-
-        let homestar_proc2 = Command::new(BIN.as_os_str())
-        .env("RUST_BACKTRACE", "0")
-        .env(
-            "RUST_LOG",
-            "homestar=debug,homestar_runtime=debug,libp2p=debug,libp2p_gossipsub::behaviour=debug",
-        )
-        .arg("start")
-        .arg("-c")
-        .arg(config2.filename())
-        .arg("--db")
-        .arg(&proc_info2.db_path)
-        .stdout(Stdio::piped())
-        .spawn()
-        .unwrap();
-        let proc_guard2 = ChildGuard::new(homestar_proc2);
-
-        let mut net_events = subscribe_network_events(ws_port1).await;
-        let sub = net_events.sub();
-
-        // Poll for connection established message
-        loop {
-            if let Ok(msg) = sub.next().with_timeout(Duration::from_secs(30)).await {
-                let json: serde_json::Value =
-                    serde_json::from_slice(&msg.unwrap().unwrap()).unwrap();
-
-                if json["connection_established"].is_object() {
-                    break;
-                }
-            } else {
-                panic!("Node one did not receive a connection established message in time.")
-            }
-        }
-
-        // Kill proceses.
-        let dead_proc1 = kill_homestar(proc_guard1.take(), Some(Duration::from_secs(2)));
-        let dead_proc2 = kill_homestar(proc_guard2.take(), Some(Duration::from_secs(2)));
-
-        // Retrieve logs.
-        let stdout1 = retrieve_output(dead_proc1);
-        let stdout2 = retrieve_output(dead_proc2);
-
-        // Check node one refused probe request from node two
-        let one_refused_probe_request = check_for_line_with(
-            stdout1,
-            vec![
-                "unable to probe a peer",
-                "Response(DialRefused)",
-                SECP256K1MULTIHASH,
-            ],
-        );
-
-        // Check node two received refusal from node one
-        let two_probe_request_failed = check_for_line_with(
-            stdout2,
-            vec![
-                "requested probe failed",
-                "Response(DialRefused)",
-                ED25519MULTIHASH,
-            ],
-        );
-
-        assert!(one_refused_probe_request);
-        assert!(two_probe_request_failed);
-    });
-
-    Ok(())
-}

From 8e39d285e32abf650b6ab13362d44d175d58229b Mon Sep 17 00:00:00 2001
From: Brian Ginsburg <gins@brianginsburg.com>
Date: Wed, 3 Apr 2024 10:40:51 -0700
Subject: [PATCH 12/12] chore: Probe addresses on identify to confirm them

---
 homestar-runtime/src/event_handler/swarm_event.rs | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/homestar-runtime/src/event_handler/swarm_event.rs b/homestar-runtime/src/event_handler/swarm_event.rs
index a6f879f4..b9c65f24 100644
--- a/homestar-runtime/src/event_handler/swarm_event.rs
+++ b/homestar-runtime/src/event_handler/swarm_event.rs
@@ -242,7 +242,7 @@ async fn handle_swarm_event<DB: Database>(
 
                     let num_addresses = event_handler.swarm.external_addresses().count();
 
-                    // Add observed address as an external address if we are identifying ourselves
+                    // Probe observed address as an external address if we are identifying ourselves
                     if &peer_id == event_handler.swarm.local_peer_id()
                         && num_addresses < event_handler.external_address_limit as usize
                     {
@@ -254,9 +254,15 @@ async fn handle_swarm_event<DB: Database>(
                                 _ => None,
                             })
                             .all(|proto| !proto.is_private())
-                            // Identify observed a potentially valid external address that we weren't aware of.
-                            // Add it to the addresses we announce to other peers.
-                            .then(|| event_handler.swarm.add_external_address(info.observed_addr));
+                            // We have observed a potentially valid external address that we weren't aware of.
+                            // Probe it with AutoNAT to confirm it and on confirmation add it to addresses we announce to peers.
+                            .then(|| {
+                                event_handler
+                                    .swarm
+                                    .behaviour_mut()
+                                    .autonat
+                                    .probe_address(info.observed_addr)
+                            });
                     }
 
                     let behavior = event_handler.swarm.behaviour_mut();