chore: flaky test updates (#359)

ipvm-wg · Oct 6, 2023 · b286ea0 · b286ea0
1 parent 363c941
commit b286ea0
Show file tree

Hide file tree

Showing 14 changed files with 121 additions and 46 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml
@@ -179,6 +179,7 @@ predicates = { version = "3.0", default-features = false }
 prometheus-parse = "0.2.4"
 rand = { workspace = true }
 retry = { version = "2.0", default-features = false }
+rm_rf = "0.6"
 serial_test = { version = "2.0", default-features = false, features = [
   "file_locks",
 ] }

diff --git a/homestar-runtime/tests/cli.rs b/homestar-runtime/tests/cli.rs
@@ -135,14 +135,16 @@ fn test_server_serial() -> Result<()> {
 
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
+        .arg("-c")
+        .arg("tests/fixtures/test_v6.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
         .spawn()
         .unwrap();
 
-    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9998);
+    let result = retry(Fixed::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -153,13 +155,19 @@ fn test_server_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("ping")
+        .arg("--host")
+        .arg("::1")
+        .arg("-p")
+        .arg("9998")
         .assert()
         .success()
         .stdout(predicate::str::contains("::1"))
         .stdout(predicate::str::contains("pong"));
 
     Command::new(BIN.as_os_str())
         .arg("ping")
+        .arg("--host")
+        .arg("::1")
         .arg("-p")
         .arg("9999")
         .assert()
@@ -204,7 +212,7 @@ fn test_workflow_run_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(30), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -216,7 +224,7 @@ fn test_workflow_run_serial() -> Result<()> {
     Command::new(BIN.as_os_str())
         .arg("run")
         .arg("-w")
-        .arg("./fixtures/test-workflow-add-one.json")
+        .arg("tests/fixtures/test-workflow-add-one.json")
         .assert()
         .success()
         .stdout(predicate::str::contains(
@@ -232,7 +240,7 @@ fn test_workflow_run_serial() -> Result<()> {
     Command::new(BIN.as_os_str())
         .arg("run")
         .arg("-w")
-        .arg("./fixtures/test-workflow-add-one.json")
+        .arg("tests/fixtures/test-workflow-add-one.json")
         .assert()
         .success()
         .stdout(predicate::str::contains(
@@ -272,14 +280,16 @@ fn test_daemon_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("start")
+        .arg("-c")
+        .arg("tests/fixtures/test_v4_alt.toml")
         .arg("-d")
         .env("DATABASE_URL", "homestar.db")
         .stdout(Stdio::piped())
         .assert()
         .success();
 
-    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9997);
+    let result = retry(Fixed::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -289,9 +299,13 @@ fn test_daemon_serial() -> Result<()> {
 
     Command::new(BIN.as_os_str())
         .arg("ping")
+        .arg("--host")
+        .arg("127.0.0.1")
+        .arg("-p")
+        .arg("9997")
         .assert()
         .success()
-        .stdout(predicate::str::contains("::1"))
+        .stdout(predicate::str::contains("127.0.0.1"))
         .stdout(predicate::str::contains("pong"));
 
     let _ = stop_all_bins();
@@ -318,7 +332,7 @@ fn test_signal_kill_serial() -> Result<()> {
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(10), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -364,15 +378,15 @@ fn test_server_v4_serial() -> Result<()> {
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
         .arg("-c")
-        .arg("fixtures/test_v4.toml")
+        .arg("tests/fixtures/test_v4.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
         .spawn()
         .unwrap();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(30), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 
@@ -421,15 +435,15 @@ fn test_daemon_v4_serial() -> Result<()> {
     Command::new(BIN.as_os_str())
         .arg("start")
         .arg("-c")
-        .arg("fixtures/test_v4.toml")
+        .arg("tests/fixtures/test_v4.toml")
         .arg("-d")
         .env("DATABASE_URL", "homestar.db")
         .stdout(Stdio::piped())
         .assert()
         .success();
 
     let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999);
-    let result = retry(Fixed::from_millis(500), || {
+    let result = retry(Fixed::from_millis(1000).take(30), || {
         TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
     });
 

diff --git a/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml
@@ -0,0 +1,12 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4004
+
+[node]
+
+[node.network]
+rpc_port = 3031
+websocket_port = 9090
+
+[node.network.keypair_config]
+existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }
diff --git a/homestar-runtime/tests/fixtures/network_node1/config/settings.toml b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml
@@ -0,0 +1,16 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4002
+
+[node]
+
+[node.network]
+rpc_port = 3032
+websocket_port = 9092
+listen_address = "/ip4/127.0.0.1/tcp/7000"
+node_addresses = [
+  "/ip4/127.0.0.1/tcp/7001/p2p/16Uiu2HAm3g9AomQNeEctL2hPwLapap7AtPSNt8ZrBny4rLx1W5Dc",
+]
+
+[node.network.keypair_config]
+existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" }
diff --git a/homestar-runtime/tests/fixtures/network_node2/config/settings.toml b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml
@@ -0,0 +1,16 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4003
+
+[node]
+
+[node.network]
+rpc_port = 3033
+websocket_port = 9093
+listen_address = "/ip4/127.0.0.1/tcp/7001"
+node_addresses = [
+  "/ip4/127.0.0.1/tcp/7000/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN",
+]
+
+[node.network.keypair_config]
+existing = { key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" }
diff --git a/...ntime/fixtures/test-workflow-add-one.json → ...tests/fixtures/test-workflow-add-one.json b/...ntime/fixtures/test-workflow-add-one.json → ...tests/fixtures/test-workflow-add-one.json
diff --git a/homestar-runtime/fixtures/test_v4.toml → homestar-runtime/tests/fixtures/test_v4.toml b/homestar-runtime/fixtures/test_v4.toml → homestar-runtime/tests/fixtures/test_v4.toml
diff --git a/homestar-runtime/tests/fixtures/test_v4_alt.toml b/homestar-runtime/tests/fixtures/test_v4_alt.toml
@@ -0,0 +1,10 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4007
+
+[node]
+
+[node.network]
+events_buffer_len = 1000
+rpc_port = 9997
+rpc_host = "127.0.0.1"
diff --git a/homestar-runtime/tests/fixtures/test_v6.toml b/homestar-runtime/tests/fixtures/test_v6.toml
@@ -0,0 +1,10 @@
+[monitoring]
+process_collector_interval = 500
+metrics_port = 4006
+
+[node]
+
+[node.network]
+events_buffer_len = 1000
+rpc_port = 9998
+rpc_host = "::1"
diff --git a/homestar-runtime/tests/metrics.rs b/homestar-runtime/tests/metrics.rs
@@ -5,21 +5,22 @@ use reqwest::StatusCode;
 use retry::{delay::Fixed, retry, OperationResult};
 use serial_test::file_serial;
 use std::{
+    net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream},
     path::PathBuf,
     process::{Command, Stdio},
     time::Duration,
 };
 use wait_timeout::ChildExt;
 
 static BIN: Lazy<PathBuf> = Lazy::new(|| assert_cmd::cargo::cargo_bin(BIN_NAME));
-const METRICS_URL: &str = "http://localhost:4000";
+const METRICS_URL: &str = "http://localhost:4004";
 
 #[test]
 #[file_serial]
 fn test_metrics_serial() -> Result<()> {
     fn sample_metrics() -> prometheus_parse::Value {
         let body = retry(
-            Fixed::from_millis(500).take(2),
+            Fixed::from_millis(1000).take(10),
             || match reqwest::blocking::get(METRICS_URL) {
                 Ok(response) => match response.status() {
                     StatusCode::OK => OperationResult::Ok(response.text()),
@@ -28,10 +29,9 @@ fn test_metrics_serial() -> Result<()> {
                 Err(_) => OperationResult::Retry("Metrics server not available"),
             },
         )
-        .unwrap()
-        .expect("Metrics server failed to serve metrics");
+        .unwrap();
 
-        let lines: Vec<_> = body.lines().map(|s| Ok(s.to_owned())).collect();
+        let lines: Vec<_> = body.unwrap().lines().map(|s| Ok(s.to_owned())).collect();
         let metrics =
             prometheus_parse::Scrape::parse(lines.into_iter()).expect("Unable to parse metrics");
 
@@ -49,16 +49,26 @@ fn test_metrics_serial() -> Result<()> {
     let mut homestar_proc = Command::new(BIN.as_os_str())
         .arg("start")
         .arg("-c")
-        .arg("tests/test_node2/config/settings.toml")
+        .arg("tests/fixtures/metrics_node/config/settings.toml")
         .arg("--db")
         .arg("homestar.db")
         .stdout(Stdio::piped())
         .spawn()
         .unwrap();
 
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4004);
+    let result = retry(Fixed::from_millis(1000).take(10), || {
+        TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both))
+    });
+
+    if result.is_err() {
+        homestar_proc.kill().unwrap();
+        panic!("Homestar server/runtime failed to start in time");
+    }
+
     let sample1 = sample_metrics();
 
-    let sample2 = retry(Fixed::from_millis(500).take(5), || {
+    let sample2 = retry(Fixed::from_millis(500).take(3), || {
         let sample2 = sample_metrics();
         if sample1 != sample2 {
             OperationResult::Ok(sample2)

diff --git a/homestar-runtime/tests/test_node1/config/settings.toml b/homestar-runtime/tests/test_node1/config/settings.toml
diff --git a/homestar-runtime/tests/test_node2/config/settings.toml b/homestar-runtime/tests/test_node2/config/settings.toml
diff --git a/homestar-runtime/tests/utils.rs b/homestar-runtime/tests/utils.rs
@@ -67,6 +67,7 @@ pub(crate) fn stop_ipfs() -> Result<()> {
         .stderr(Stdio::null())
         .status()
         .context("failed to stop IPFS daemon")?;
+    rm_rf::ensure_removed(path).unwrap();
 
     Ok(())
 }
@@ -97,7 +98,7 @@ pub(crate) fn kill_homestar_process() -> Result<()> {
         );
 
     let _result = signal::kill(Pid::from_raw(pid.try_into().unwrap()), Signal::SIGTERM);
-    let _result = retry(Fixed::from_millis(500), || {
+    let _result = retry(Fixed::from_millis(1000).take(5), || {
         Command::new(BIN.as_os_str())
             .arg("ping")
             .assert()