From b286ea091ffebc76e8c230982f06b12bc602fef8 Mon Sep 17 00:00:00 2001
From: Zeeshan Lakhani <zeeshan@fission.codes>
Date: Fri, 6 Oct 2023 15:13:44 -0400
Subject: [PATCH 1/2] chore: flaky test updates (#359)

---
 Cargo.lock                                    | 10 +++++
 homestar-runtime/Cargo.toml                   |  1 +
 homestar-runtime/tests/cli.rs                 | 40 +++++++++++++------
 .../metrics_node/config/settings.toml         | 12 ++++++
 .../network_node1/config/settings.toml        | 16 ++++++++
 .../network_node2/config/settings.toml        | 16 ++++++++
 .../fixtures/test-workflow-add-one.json       |  0
 .../{ => tests}/fixtures/test_v4.toml         |  0
 .../tests/fixtures/test_v4_alt.toml           | 10 +++++
 homestar-runtime/tests/fixtures/test_v6.toml  | 10 +++++
 homestar-runtime/tests/metrics.rs             | 24 +++++++----
 .../tests/test_node1/config/settings.toml     | 13 ------
 .../tests/test_node2/config/settings.toml     | 12 ------
 homestar-runtime/tests/utils.rs               |  3 +-
 14 files changed, 121 insertions(+), 46 deletions(-)
 create mode 100644 homestar-runtime/tests/fixtures/metrics_node/config/settings.toml
 create mode 100644 homestar-runtime/tests/fixtures/network_node1/config/settings.toml
 create mode 100644 homestar-runtime/tests/fixtures/network_node2/config/settings.toml
 rename homestar-runtime/{ => tests}/fixtures/test-workflow-add-one.json (100%)
 rename homestar-runtime/{ => tests}/fixtures/test_v4.toml (100%)
 create mode 100644 homestar-runtime/tests/fixtures/test_v4_alt.toml
 create mode 100644 homestar-runtime/tests/fixtures/test_v6.toml
 delete mode 100644 homestar-runtime/tests/test_node1/config/settings.toml
 delete mode 100644 homestar-runtime/tests/test_node2/config/settings.toml

diff --git a/Cargo.lock b/Cargo.lock
index 886af4cd..d35ec5af 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2383,6 +2383,7 @@ dependencies = [
  "rand",
  "reqwest",
  "retry",
+ "rm_rf",
  "sec1",
  "semver",
  "serde",
@@ -5043,6 +5044,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rm_rf"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3443b7a35aa12ed2e99edfc0ecbefe6a53b4848305cc83e29981dfa1aea1f71e"
+dependencies = [
+ "stacker",
+]
+
 [[package]]
 name = "rmp"
 version = "0.8.12"
diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml
index 7fd4c12b..775eafda 100644
--- a/homestar-runtime/Cargo.toml
+++ b/homestar-runtime/Cargo.toml
@@ -179,6 +179,7 @@ predicates = { version = "3.0", default-features = false }
 prometheus-parse = "0.2.4"
 rand = { workspace = true }
 retry = { version = "2.0", default-features = false }
+rm_rf = "0.6"
 serial_test = { version = "2.0", default-features = false, features = [
   "file_locks",
 ] }
diff --git a/homestar-runtime/tests/cli.rs b/homestar-runtime/tests/cli.rs
index aaba16a9..dc0c6c6b 100644
--- a/homestar-runtime/tests/cli.rs
+++ b/homestar-runtime/tests/cli.rs
@@ -135,14 +135,16 @@ fn test_server_serial() -> Result<()> {
 
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
+        .arg("-c")
+        .arg("tests/fixtures/test_v6.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
         .spawn()
         .unwrap();
 
-    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9998);
+    let result = retry(Fixed::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -153,6 +155,10 @@ fn test_server_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("ping")
+        .arg("--host")
+        .arg("::1")
+        .arg("-p")
+        .arg("9998")
         .assert()
         .success()
         .stdout(predicate::str::contains("::1"))
@@ -160,6 +166,8 @@ fn test_server_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("ping")
+        .arg("--host")
+        .arg("::1")
         .arg("-p")
         .arg("9999")
         .assert()
@@ -204,7 +212,7 @@ fn test_workflow_run_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(30), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -216,7 +224,7 @@ fn test_workflow_run_serial() -> Result<()> {
     Command::new(BIN.as_os_str())
         .arg("run")
         .arg("-w")
-        .arg("./fixtures/test-workflow-add-one.json")
+        .arg("tests/fixtures/test-workflow-add-one.json")
         .assert()
         .success()
         .stdout(predicate::str::contains(
@@ -232,7 +240,7 @@ fn test_workflow_run_serial() -> Result<()> {
     Command::new(BIN.as_os_str())
         .arg("run")
         .arg("-w")
-        .arg("./fixtures/test-workflow-add-one.json")
+        .arg("tests/fixtures/test-workflow-add-one.json")
         .assert()
         .success()
         .stdout(predicate::str::contains(
@@ -272,14 +280,16 @@ fn test_daemon_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("start")
+        .arg("-c")
+        .arg("tests/fixtures/test_v4_alt.toml")
         .arg("-d")
         .env("DATABASE_URL", "homestar.db")
         .stdout(Stdio::piped())
         .assert()
         .success();
 
-    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9997);
+    let result = retry(Fixed::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -289,9 +299,13 @@ fn test_daemon_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("ping")
+        .arg("--host")
+        .arg("127.0.0.1")
+        .arg("-p")
+        .arg("9997")
         .assert()
         .success()
-        .stdout(predicate::str::contains("::1"))
+        .stdout(predicate::str::contains("127.0.0.1"))
         .stdout(predicate::str::contains("pong"));
 
     let _ = stop_all_bins();
@@ -318,7 +332,7 @@ fn test_signal_kill_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -364,7 +378,7 @@ fn test_server_v4_serial() -> Result<()> {
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
         .arg("-c")
-        .arg("fixtures/test_v4.toml")
+        .arg("tests/fixtures/test_v4.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
@@ -372,7 +386,7 @@ fn test_server_v4_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(30), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -421,7 +435,7 @@ fn test_daemon_v4_serial() -> Result<()> {
     Command::new(BIN.as_os_str())
         .arg("start")
         .arg("-c")
-        .arg("fixtures/test_v4.toml")
+        .arg("tests/fixtures/test_v4.toml")
         .arg("-d")
         .env("DATABASE_URL", "homestar.db")
         .stdout(Stdio::piped())
@@ -429,7 +443,7 @@ fn test_daemon_v4_serial() -> Result<()> {
         .success();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(30), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
diff --git a/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml
new file mode 100644
index 00000000..c8f2f4b4
--- /dev/null
+++ b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml
@@ -0,0 +1,12 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4004
+
+[node]
+
+[node.network]
+rpc_port = 3031
+websocket_port = 9090
+
+[node.network.keypair_config]
+existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }
diff --git a/homestar-runtime/tests/fixtures/network_node1/config/settings.toml b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml
new file mode 100644
index 00000000..33090934
--- /dev/null
+++ b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml
@@ -0,0 +1,16 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4002
+
+[node]
+
+[node.network]
+rpc_port = 3032
+websocket_port = 9092
+listen_address = "/ip4/127.0.0.1/tcp/7000"
+node_addresses = [
+  "/ip4/127.0.0.1/tcp/7001/p2p/16Uiu2HAm3g9AomQNeEctL2hPwLapap7AtPSNt8ZrBny4rLx1W5Dc",
+]
+
+[node.network.keypair_config]
+existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }
diff --git a/homestar-runtime/tests/fixtures/network_node2/config/settings.toml b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml
new file mode 100644
index 00000000..b1cf1e0a
--- /dev/null
+++ b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml
@@ -0,0 +1,16 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4003
+
+[node]
+
+[node.network]
+rpc_port = 3033
+websocket_port = 9093
+listen_address = "/ip4/127.0.0.1/tcp/7001"
+node_addresses = [
+  "/ip4/127.0.0.1/tcp/7000/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN",
+]
+
+[node.network.keypair_config]
+existing = { key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" }
diff --git a/homestar-runtime/fixtures/test-workflow-add-one.json b/homestar-runtime/tests/fixtures/test-workflow-add-one.json
similarity index 100%
rename from homestar-runtime/fixtures/test-workflow-add-one.json
rename to homestar-runtime/tests/fixtures/test-workflow-add-one.json
diff --git a/homestar-runtime/fixtures/test_v4.toml b/homestar-runtime/tests/fixtures/test_v4.toml
similarity index 100%
rename from homestar-runtime/fixtures/test_v4.toml
rename to homestar-runtime/tests/fixtures/test_v4.toml
diff --git a/homestar-runtime/tests/fixtures/test_v4_alt.toml b/homestar-runtime/tests/fixtures/test_v4_alt.toml
new file mode 100644
index 00000000..11a3c23f
--- /dev/null
+++ b/homestar-runtime/tests/fixtures/test_v4_alt.toml
@@ -0,0 +1,10 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4007
+
+[node]
+
+[node.network]
+events_buffer_len = 1000
+rpc_port = 9997
+rpc_host = "127.0.0.1"
diff --git a/homestar-runtime/tests/fixtures/test_v6.toml b/homestar-runtime/tests/fixtures/test_v6.toml
new file mode 100644
index 00000000..2e35703f
--- /dev/null
+++ b/homestar-runtime/tests/fixtures/test_v6.toml
@@ -0,0 +1,10 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4006
+
+[node]
+
+[node.network]
+events_buffer_len = 1000
+rpc_port = 9998
+rpc_host = "::1"
diff --git a/homestar-runtime/tests/metrics.rs b/homestar-runtime/tests/metrics.rs
index 7c70209f..889b5dc1 100644
--- a/homestar-runtime/tests/metrics.rs
+++ b/homestar-runtime/tests/metrics.rs
@@ -5,6 +5,7 @@ use reqwest::StatusCode;
 use retry::{delay::Fixed, retry, OperationResult};
 use serial_test::file_serial;
 use std::{
+    net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream},
     path::PathBuf,
     process::{Command, Stdio},
     time::Duration,
@@ -12,14 +13,14 @@ use std::{
 use wait_timeout::ChildExt;
 
 static BIN: Lazy<PathBuf> = Lazy::new(|| assert_cmd::cargo::cargo_bin(BIN_NAME));
-const METRICS_URL: &str = "http://localhost:4000";
+const METRICS_URL: &str = "http://localhost:4004";
 
 #[test]
 #[file_serial]
 fn test_metrics_serial() -> Result<()> {
     fn sample_metrics() -> prometheus_parse::Value {
         let body = retry(
-            Fixed::from_millis(500).take(2),
+            Fixed::from_millis(1000).take(10),
             || match reqwest::blocking::get(METRICS_URL) {
                 Ok(response) => match response.status() {
                     StatusCode::OK => OperationResult::Ok(response.text()),
@@ -28,10 +29,9 @@ fn test_metrics_serial() -> Result<()> {
                 Err(_) => OperationResult::Retry("Metrics server not available"),
             },
         )
-        .unwrap()
-        .expect("Metrics server failed to serve metrics");
+        .unwrap();
 
-        let lines: Vec<_> = body.lines().map(|s| Ok(s.to_owned())).collect();
+        let lines: Vec<_> = body.unwrap().lines().map(|s| Ok(s.to_owned())).collect();
         let metrics =
             prometheus_parse::Scrape::parse(lines.into_iter()).expect("Unable to parse metrics");
 
@@ -49,16 +49,26 @@ fn test_metrics_serial() -> Result<()> {
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
         .arg("-c")
-        .arg("tests/test_node2/config/settings.toml")
+        .arg("tests/fixtures/metrics_node/config/settings.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
         .spawn()
         .unwrap();
 
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4004);
+    let result = retry(Fixed::from_millis(1000).take(10), || {
+        TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
+    });
+
+    if result.is_err() {
+        homestar_proc.kill().unwrap();
+        panic!("Homestar server/runtime failed to start in time");
+    }
+
     let sample1 = sample_metrics();
 
-    let sample2 = retry(Fixed::from_millis(500).take(5), || {
+    let sample2 = retry(Fixed::from_millis(500).take(3), || {
         let sample2 = sample_metrics();
         if sample1 != sample2 {
             OperationResult::Ok(sample2)
diff --git a/homestar-runtime/tests/test_node1/config/settings.toml b/homestar-runtime/tests/test_node1/config/settings.toml
deleted file mode 100644
index 305694c9..00000000
--- a/homestar-runtime/tests/test_node1/config/settings.toml
+++ /dev/null
@@ -1,13 +0,0 @@
-[monitoring]
-process_collector_interval = 500
-metrics_port = 4000
-
-[node]
-
-[node.network]
-rpc_port = 3031
-websocket_port = 9091
-node_addresses = ["/ip4/127.0.0.1/tcp/9092/ws"]
-
-[node.network.keypair_config]
-existing = { key_type = "ed25519", path = "../../fixtures/__testkey_ed25519.pem" }
diff --git a/homestar-runtime/tests/test_node2/config/settings.toml b/homestar-runtime/tests/test_node2/config/settings.toml
deleted file mode 100644
index 92570f77..00000000
--- a/homestar-runtime/tests/test_node2/config/settings.toml
+++ /dev/null
@@ -1,12 +0,0 @@
-[monitoring]
-process_collector_interval = 500
-metrics_port = 4000
-
-[node]
-
-[node.network]
-rpc_port = 3032
-websocket_port = 9092
-node_addresses = [
-  "/ip4/127.0.0.1/tcp/9091/ws/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN",
-]
diff --git a/homestar-runtime/tests/utils.rs b/homestar-runtime/tests/utils.rs
index 43d94452..afedace7 100644
--- a/homestar-runtime/tests/utils.rs
+++ b/homestar-runtime/tests/utils.rs
@@ -67,6 +67,7 @@ pub(crate) fn stop_ipfs() -> Result<()> {
         .stderr(Stdio::null())
         .status()
         .context("failed to stop IPFS daemon")?;
+    rm_rf::ensure_removed(path).unwrap();
 
     Ok(())
 }
@@ -97,7 +98,7 @@ pub(crate) fn kill_homestar_process() -> Result<()> {
         );
 
     let _result = signal::kill(Pid::from_raw(pid.try_into().unwrap()), Signal::SIGTERM);
-    let _result = retry(Fixed::from_millis(500), || {
+    let _result = retry(Fixed::from_millis(1000).take(5), || {
         Command::new(BIN.as_os_str())
             .arg("ping")
             .assert()

From 0eeb0bfb11c7d8a9c72ba4fcea445218ae8d3d30 Mon Sep 17 00:00:00 2001
From: Zeeshan Lakhani <zeeshan@fission.codes>
Date: Mon, 9 Oct 2023 14:50:08 -0400
Subject: [PATCH 2/2] feat: builds, packages, pushing contains to ghcr (#360)

Closes https://github.com/ipvm-wg/homestar/issues/276. We'll open a new
issue for more specific packages: brew, arch-linux, wix/windows. This
will need some further testing to be absolutely sure, cc @walkah.

Includes:
  - Fixes bug with empty check on annouce_addresses
  - local cross-rs setup for builds
  - github action workflows: builds.yml tied to release
* builds binaries (cross if need be) for aarch64-unknown-linux-gnu,
aarch64-unknown-linux-musl, aarch64-apple-darwin,
x86_64-unknown-linux-gnu, x86_64-unknown-linux-musl,
x86_64-apple-darwin, x86_64-pc-windows-msvc, and x86_64-unknown-freebsd
(for now)
    * builds packages for .deb, .rpm (sets out todos for brew, wix, aur)
    * docker push to ghcr.io and tag mult. versions
---
 .github/workflows/builds.yml                  | 188 ++++++++++++++++++
 .github/workflows/coverage.yml                |   7 +-
 .github/workflows/docker.yml                  |   2 +-
 Cross.toml                                    |  19 ++
 README.md                                     |   2 +-
 docker/Dockerfile                             |   3 +-
 flake.lock                                    |  18 +-
 flake.nix                                     |  12 +-
 homestar-runtime/Cargo.toml                   |  38 +++-
 homestar-runtime/src/network/swarm.rs         |   2 +-
 homestar-runtime/tests/cli.rs                 |  30 +--
 homestar-runtime/tests/fixtures/test_v4.toml  |   4 +
 .../tests/fixtures/test_v4_alt.toml           |   2 +-
 homestar-runtime/tests/fixtures/test_v6.toml  |   2 +-
 .../tests/fixtures/test_workflow.toml         |   9 +
 homestar-runtime/tests/metrics.rs             |  18 +-
 16 files changed, 317 insertions(+), 39 deletions(-)
 create mode 100644 .github/workflows/builds.yml
 create mode 100644 Cross.toml
 create mode 100644 homestar-runtime/tests/fixtures/test_workflow.toml

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
new file mode 100644
index 00000000..b30ceff2
--- /dev/null
+++ b/.github/workflows/builds.yml
@@ -0,0 +1,188 @@
+name: ⚃ Builds
+
+on:
+  workflow_dispatch:
+    inputs:
+      force-builds:
+        required: true
+        type: boolean
+        description: Publish Builds at Anytime
+
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  binary-builds:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target: aarch64-unknown-linux-gnu
+          - target: aarch64-unknown-linux-musl
+          - target: aarch64-apple-darwin
+            os: macos-latest
+          - target: x86_64-unknown-linux-gnu
+          - target: x86_64-unknown-linux-musl
+          - target: x86_64-apple-darwin
+            os: macos-latest
+          - target: x86_64-pc-windows-msvc
+            os: windows-latest
+          - target: x86_64-unknown-freebsd
+
+    permissions:
+      contents: write
+
+    timeout-minutes: 60
+    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install musl-tools
+        run: sudo apt update && sudo apt install -y musl-dev musl-tools
+        if: matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'
+
+      - name: Cache Project
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          shared-key: check-${{ matrix.target }}-${{ matrix.os }}
+
+      - name: Install Rust Toolchain
+        id: toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Override rust-toolchain.toml
+        run: rustup override set ${{steps.toolchain.outputs.name}}
+
+      - name: cross-build
+        uses: taiki-e/setup-cross-toolchain-action@v1
+        with:
+          target: ${{ matrix.target }}
+
+      - name: crt-static
+        run: echo "RUSTFLAGS=${RUSTFLAGS} -C target-feature=+crt-static" >>"${GITHUB_ENV}"
+        if: endsWith(matrix.target, 'windows-msvc')
+
+      - name: Publish Binary in Release
+        uses: taiki-e/upload-rust-binary-action@v1
+        with:
+          bin: homestar
+          target: ${{ matrix.target }}
+          tar: all
+          zip: windows
+          include: LICENSE,README.md
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  build-packages:
+    runs-on: ubuntu-latest
+
+    env:
+      LINUX_TARGET: x86_64-unknown-linux-musl
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Cache Project
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          shared-key: check-${{ env.LINUX_TARGET }}-ubuntu-latest
+
+      - name: Install musl-tools
+        run: sudo apt update && sudo apt install -y musl-dev musl-tools
+
+      - name: Install cargo-deb
+        uses: taiki-e/cache-cargo-install-action@v1
+        with:
+          tool: cargo-deb
+
+      - name: Install cargo-generate-rpm
+        uses: taiki-e/cache-cargo-install-action@v1
+        with:
+          tool: cargo-generate-rpm
+
+      - name: Create .deb
+        run: cargo deb -p homestar-runtime --target ${{env.LINUX_TARGET}}
+
+      - name: Create .rpm
+        run: cargo generate-rpm
+
+      - name: Publish Release Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.target }}
+          path: |
+            target/generate-rpm/
+            target/${{env.LINUX_TARGET}}/debian/
+          if-no-files-found: error
+
+      - name: Publish Package in Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: |
+            target/${{env.LINUX_TARGET}}/debian/*.deb
+            target/generate-rpm/*.rpm
+
+      # TODO: brew formula (Macs), cargo-wix (Windows Installs), cargo-aur (Arch)
+
+  docker-build:
+    runs-on: ubuntu-latest
+
+    env:
+      DOCKER_BUILDKIT: '1'
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    permissions:
+      contents: read
+      id-token: write
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Setup Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-flags: "--debug"
+
+      - name: Metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern=v{{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=latest
+            type=sha
+
+      - name: Docker Build & Push
+        uses: docker/build-push-action@v5
+        with:
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+          file: docker/Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 285acf43..5658f422 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -75,8 +75,11 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
 
-      - name: Generate Code Coverage
-        run: cargo llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
+      - name: Generate code coverage (nextest)
+        run: cargo llvm-cov nextest --all-features --workspace --profile=ci --lcov --output-path lcov.info
 
       - name: Upload to codecov.io
         uses: codecov/codecov-action@v3
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b6a3175b..3c02e12c 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -32,7 +32,7 @@ jobs:
           cache-to: type=gha,mode=max
           file: docker/Dockerfile
           context: .
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: false
           tags: |
             ${{ github.repository_owner }}/homestar:latest
diff --git a/Cross.toml b/Cross.toml
new file mode 100644
index 00000000..688d1ac7
--- /dev/null
+++ b/Cross.toml
@@ -0,0 +1,19 @@
+[build.env]
+passthrough = [
+  "CARGO_INCREMENTAL",
+  "RUST_BACKTRACE",
+  "CARGO_TERM_COLOR",
+  "RUSTFLAGS",
+]
+
+[target.x86_64-unknown-linux-musl]
+image = "burntsushi/cross:x86_64-unknown-linux-musl"
+
+[target.aarch64-unknown-linux-musl]
+image = "burntsushi/cross:aarch64-unknown-linux-gnu"
+
+[target.x86_64-apple-darwin]
+image = "freeznet/x86_64-apple-darwin-cross:11.3"
+
+[target.aarch64-apple-darwin]
+image = "freeznet/aarch64-apple-darwin-cross:11.3"
diff --git a/README.md b/README.md
index 3a1b0674..58ea6072 100644
--- a/README.md
+++ b/README.md
@@ -63,7 +63,7 @@
 If you're looking to help develop `homestar`, please dive right into our
 [development](./DEVELOPMENT.md) guide.
 
-Otherwise, the easiest way to get started and "see" `homestar` in action is to
+Otherwise, the easiest way to get started and see `homestar` in action is to
 follow-along and run our image-processing
 [websocket relay](./examples/websocket-relay) example, which integrates
 `homestar` with a browser application to run a
diff --git a/docker/Dockerfile b/docker/Dockerfile
index f0d29caa..53097d50 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -40,8 +40,7 @@ ENTRYPOINT ["/bin/bash"]
 # cache depencies
 RUN mkdir .cargo
 RUN cargo vendor > .cargo/config
-RUN --mount=type=cache,target=$CARGO_HOME/registry \
-    cargo install diesel_cli --no-default-features --features "sqlite-bundled"
+RUN cargo install diesel_cli --no-default-features --features "sqlite-bundled"
 RUN diesel setup --database-url $DATABASE_URL
 RUN diesel migration run --migration-dir ./homestar-runtime/migrations
 RUN --mount=type=cache,id=cargo,target=$CARGO_HOME/registry \
diff --git a/flake.lock b/flake.lock
index 0514ad76..edcb403f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -3,11 +3,11 @@
     "flake-compat": {
       "flake": false,
       "locked": {
-        "lastModified": 1696267196,
-        "narHash": "sha256-AAQ/2sD+0D18bb8hKuEEVpHUYD1GmO2Uh/taFamn6XQ=",
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "4f910c9827911b1ec2bf26b5a062cd09f8d89f85",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
         "type": "github"
       },
       "original": {
@@ -36,11 +36,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1696234590,
-        "narHash": "sha256-mgOzQYTvaTT4bFopVOadlndy2RPwLy60rDjIWOGujwo=",
+        "lastModified": 1696577711,
+        "narHash": "sha256-94VRjvClIKDym1QRqPkX5LTQoAwZ1E6QE/3dWtOXSIQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f902cb49892d300ff15cb237e48aa1cad79d68c3",
+        "rev": "a2eb207f45e4a14a1e3019d9e3863d1e208e2295",
         "type": "github"
       },
       "original": {
@@ -68,11 +68,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1696212720,
-        "narHash": "sha256-kuI/n1LpXv9S1wXfIzvYJudBfQeF0YGS4G8xJQQxM/M=",
+        "lastModified": 1696558324,
+        "narHash": "sha256-TnnP4LGwDB8ZGE7h2n4nA9Faee8xPkMdNcyrzJ57cbw=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "1aaa2dc3e7367f2014f939c927e9e768a0cc2f08",
+        "rev": "fdb37574a04df04aaa8cf7708f94a9309caebe2b",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index 0069f766..1b4fe582 100644
--- a/flake.nix
+++ b/flake.nix
@@ -32,7 +32,14 @@
 
         rust-toolchain = (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml).override {
           extensions = ["cargo" "clippy" "rustfmt" "rust-src" "rust-std"];
-          targets = ["wasm32-unknown-unknown" "wasm32-wasi"];
+          targets = [
+            "wasm32-unknown-unknown"
+            "wasm32-wasi"
+            "x86_64-apple-darwin"
+            "aarch64-apple-darwin"
+            "x86_64-unknown-linux-musl"
+            "aarch64-unknown-linux-musl"
+          ];
         };
 
         nightly-rustfmt = pkgs.rust-bin.nightly.latest.rustfmt;
@@ -45,6 +52,8 @@
 
         cargo-installs = with pkgs; [
           cargo-deny
+          cargo-deb
+          cargo-cross
           cargo-expand
           cargo-nextest
           cargo-outdated
@@ -53,6 +62,7 @@
           cargo-unused-features
           cargo-udeps
           cargo-watch
+          rustup
           twiggy
           wasm-tools
         ];
diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml
index 775eafda..abbe18c1 100644
--- a/homestar-runtime/Cargo.toml
+++ b/homestar-runtime/Cargo.toml
@@ -23,7 +23,6 @@ autotests = false
 path = "src/lib.rs"
 bench = false
 doctest = true
-crate-type = ["cdylib", "rlib"]
 
 [[bin]]
 name = "homestar"
@@ -204,3 +203,40 @@ websocket-server = ["dep:axum"]
 all-features = true
 # defines the configuration attribute `docsrs`
 rustdoc-args = ["--cfg", "docsrs"]
+
+[package.metadata.deb]
+maintainer = "James Walker <james@fission.codes>"
+license-file = ["LICENSE", "0"]
+extended-description-file = "README.md"
+section = "network"
+priority = "optional"
+assets = [
+  [
+    "../target/x86_64-unknown-linux-musl/release/homestar",
+    "usr/bin/",
+    "755",
+  ],
+  [
+    "../CHANGELOG.md",
+    "usr/share/doc/homestar/",
+    "644",
+  ],
+  [
+    "../LICENSE",
+    "usr/share/doc/homestar/",
+    "644",
+  ],
+  [
+    "../README.md",
+    "usr/share/doc/homestar/",
+    "644",
+  ],
+]
+
+[package.metadata.generate-rpm]
+assets = [
+  { source = "../target/x86_64-unknown-linux-musl/release/homestar", dest = "/usr/bin/homestar", mode = "755" },
+  { source = "../CHANGELOG.md", dest = "/usr/share/doc/homestar/CHANGELOG.md", mode = "644", doc = true },
+  { source = "../LICENSE", dest = "/usr/share/doc/homestar/LICENSE.md", mode = "644", doc = true },
+  { source = "../README.md", dest = "/usr/share/doc/homestar/README.md", mode = "644", doc = true },
+]
diff --git a/homestar-runtime/src/network/swarm.rs b/homestar-runtime/src/network/swarm.rs
index 72837161..0ec8ed13 100644
--- a/homestar-runtime/src/network/swarm.rs
+++ b/homestar-runtime/src/network/swarm.rs
@@ -120,7 +120,7 @@ pub(crate) fn init(
     swarm.listen_on(settings.listen_address.to_string().parse()?)?;
 
     // add external addresses from settings
-    if settings.announce_addresses.is_empty() {
+    if !settings.announce_addresses.is_empty() {
         for addr in settings.announce_addresses.iter() {
             swarm.add_external_address(addr.clone());
         }
diff --git a/homestar-runtime/tests/cli.rs b/homestar-runtime/tests/cli.rs
index dc0c6c6b..2fcc1b05 100644
--- a/homestar-runtime/tests/cli.rs
+++ b/homestar-runtime/tests/cli.rs
@@ -5,7 +5,7 @@ use anyhow::Result;
 use assert_cmd::prelude::*;
 use once_cell::sync::Lazy;
 use predicates::prelude::*;
-use retry::{delay::Fixed, retry};
+use retry::{delay::Exponential, retry};
 use serial_test::file_serial;
 use std::{
     net::{IpAddr, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, TcpStream},
@@ -143,8 +143,8 @@ fn test_server_serial() -> Result<()> {
         .spawn()
         .unwrap();
 
-    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9998);
-    let result = retry(Fixed::from_millis(1000).take(10), || {
+    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9991);
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -158,7 +158,7 @@ fn test_server_serial() -> Result<()> {
         .arg("--host")
         .arg("::1")
         .arg("-p")
-        .arg("9998")
+        .arg("9991")
         .assert()
         .success()
         .stdout(predicate::str::contains("::1"))
@@ -205,14 +205,16 @@ fn test_workflow_run_serial() -> Result<()> {
 
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
+        .arg("-c")
+        .arg("tests/fixtures/test_workflow.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
         .spawn()
         .unwrap();
 
-    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(1000).take(30), || {
+    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9888);
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -223,6 +225,8 @@ fn test_workflow_run_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("run")
+        .arg("-p")
+        .arg("9888")
         .arg("-w")
         .arg("tests/fixtures/test-workflow-add-one.json")
         .assert()
@@ -239,6 +243,8 @@ fn test_workflow_run_serial() -> Result<()> {
     // run another one of the same!
     Command::new(BIN.as_os_str())
         .arg("run")
+        .arg("-p")
+        .arg("9888")
         .arg("-w")
         .arg("tests/fixtures/test-workflow-add-one.json")
         .assert()
@@ -288,8 +294,8 @@ fn test_daemon_serial() -> Result<()> {
         .assert()
         .success();
 
-    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9997);
-    let result = retry(Fixed::from_millis(1000).take(10), || {
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9987);
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -302,7 +308,7 @@ fn test_daemon_serial() -> Result<()> {
         .arg("--host")
         .arg("127.0.0.1")
         .arg("-p")
-        .arg("9997")
+        .arg("9987")
         .assert()
         .success()
         .stdout(predicate::str::contains("127.0.0.1"))
@@ -332,7 +338,7 @@ fn test_signal_kill_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(1000).take(10), || {
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -386,7 +392,7 @@ fn test_server_v4_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999);
-    let result = retry(Fixed::from_millis(1000).take(30), || {
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -443,7 +449,7 @@ fn test_daemon_v4_serial() -> Result<()> {
         .success();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999);
-    let result = retry(Fixed::from_millis(1000).take(30), || {
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
diff --git a/homestar-runtime/tests/fixtures/test_v4.toml b/homestar-runtime/tests/fixtures/test_v4.toml
index 37bdce12..f4536b6c 100644
--- a/homestar-runtime/tests/fixtures/test_v4.toml
+++ b/homestar-runtime/tests/fixtures/test_v4.toml
@@ -1,3 +1,7 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4010
+
 [node]
 
 [node.network]
diff --git a/homestar-runtime/tests/fixtures/test_v4_alt.toml b/homestar-runtime/tests/fixtures/test_v4_alt.toml
index 11a3c23f..f111adb9 100644
--- a/homestar-runtime/tests/fixtures/test_v4_alt.toml
+++ b/homestar-runtime/tests/fixtures/test_v4_alt.toml
@@ -6,5 +6,5 @@ metrics_port = 4007
 
 [node.network]
 events_buffer_len = 1000
-rpc_port = 9997
+rpc_port = 9987
 rpc_host = "127.0.0.1"
diff --git a/homestar-runtime/tests/fixtures/test_v6.toml b/homestar-runtime/tests/fixtures/test_v6.toml
index 2e35703f..938a30b2 100644
--- a/homestar-runtime/tests/fixtures/test_v6.toml
+++ b/homestar-runtime/tests/fixtures/test_v6.toml
@@ -6,5 +6,5 @@ metrics_port = 4006
 
 [node.network]
 events_buffer_len = 1000
-rpc_port = 9998
+rpc_port = 9991
 rpc_host = "::1"
diff --git a/homestar-runtime/tests/fixtures/test_workflow.toml b/homestar-runtime/tests/fixtures/test_workflow.toml
new file mode 100644
index 00000000..6a820d4d
--- /dev/null
+++ b/homestar-runtime/tests/fixtures/test_workflow.toml
@@ -0,0 +1,9 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4012
+
+[node]
+
+[node.network]
+events_buffer_len = 1000
+rpc_port = 9888
diff --git a/homestar-runtime/tests/metrics.rs b/homestar-runtime/tests/metrics.rs
index 889b5dc1..9081f9de 100644
--- a/homestar-runtime/tests/metrics.rs
+++ b/homestar-runtime/tests/metrics.rs
@@ -2,7 +2,7 @@ use crate::utils::{stop_homestar, BIN_NAME};
 use anyhow::Result;
 use once_cell::sync::Lazy;
 use reqwest::StatusCode;
-use retry::{delay::Fixed, retry, OperationResult};
+use retry::{delay::Exponential, retry, OperationResult};
 use serial_test::file_serial;
 use std::{
     net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream},
@@ -20,7 +20,7 @@ const METRICS_URL: &str = "http://localhost:4004";
 fn test_metrics_serial() -> Result<()> {
     fn sample_metrics() -> prometheus_parse::Value {
         let body = retry(
-            Fixed::from_millis(1000).take(10),
+            Exponential::from_millis(500).take(20),
             || match reqwest::blocking::get(METRICS_URL) {
                 Ok(response) => match response.status() {
                     StatusCode::OK => OperationResult::Ok(response.text()),
@@ -57,7 +57,7 @@ fn test_metrics_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4004);
-    let result = retry(Fixed::from_millis(1000).take(10), || {
+    let result = retry(Exponential::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -68,17 +68,21 @@ fn test_metrics_serial() -> Result<()> {
 
     let sample1 = sample_metrics();
 
-    let sample2 = retry(Fixed::from_millis(500).take(3), || {
+    let sample2 = retry(Exponential::from_millis(500).take(10), || {
         let sample2 = sample_metrics();
         if sample1 != sample2 {
             OperationResult::Ok(sample2)
         } else {
             OperationResult::Retry("Samples are the same")
         }
-    })
-    .unwrap();
+    });
+
+    if sample2.is_err() {
+        homestar_proc.kill().unwrap();
+        panic!("Could not generate a diff in sample(s)");
+    }
 
-    assert_ne!(sample1, sample2);
+    assert_ne!(sample1, sample2.unwrap());
 
     if let Ok(None) = homestar_proc.try_wait() {
         let _status_code = match homestar_proc.wait_timeout(Duration::from_secs(1)).unwrap() {