From b286ea091ffebc76e8c230982f06b12bc602fef8 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 6 Oct 2023 15:13:44 -0400 Subject: [PATCH] chore: flaky test updates (#359) --- Cargo.lock | 10 +++++ homestar-runtime/Cargo.toml | 1 + homestar-runtime/tests/cli.rs | 40 +++++++++++++------ .../metrics_node/config/settings.toml | 12 ++++++ .../network_node1/config/settings.toml | 16 ++++++++ .../network_node2/config/settings.toml | 16 ++++++++ .../fixtures/test-workflow-add-one.json | 0 .../{ => tests}/fixtures/test_v4.toml | 0 .../tests/fixtures/test_v4_alt.toml | 10 +++++ homestar-runtime/tests/fixtures/test_v6.toml | 10 +++++ homestar-runtime/tests/metrics.rs | 24 +++++++---- .../tests/test_node1/config/settings.toml | 13 ------ .../tests/test_node2/config/settings.toml | 12 ------ homestar-runtime/tests/utils.rs | 3 +- 14 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 homestar-runtime/tests/fixtures/metrics_node/config/settings.toml create mode 100644 homestar-runtime/tests/fixtures/network_node1/config/settings.toml create mode 100644 homestar-runtime/tests/fixtures/network_node2/config/settings.toml rename homestar-runtime/{ => tests}/fixtures/test-workflow-add-one.json (100%) rename homestar-runtime/{ => tests}/fixtures/test_v4.toml (100%) create mode 100644 homestar-runtime/tests/fixtures/test_v4_alt.toml create mode 100644 homestar-runtime/tests/fixtures/test_v6.toml delete mode 100644 homestar-runtime/tests/test_node1/config/settings.toml delete mode 100644 homestar-runtime/tests/test_node2/config/settings.toml diff --git a/Cargo.lock b/Cargo.lock index 886af4cd..d35ec5af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2383,6 +2383,7 @@ dependencies = [ "rand", "reqwest", "retry", + "rm_rf", "sec1", "semver", "serde", @@ -5043,6 +5044,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rm_rf" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3443b7a35aa12ed2e99edfc0ecbefe6a53b4848305cc83e29981dfa1aea1f71e" +dependencies = [ + "stacker", +] + [[package]] name = "rmp" version = "0.8.12" diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml index 7fd4c12b..775eafda 100644 --- a/homestar-runtime/Cargo.toml +++ b/homestar-runtime/Cargo.toml @@ -179,6 +179,7 @@ predicates = { version = "3.0", default-features = false } prometheus-parse = "0.2.4" rand = { workspace = true } retry = { version = "2.0", default-features = false } +rm_rf = "0.6" serial_test = { version = "2.0", default-features = false, features = [ "file_locks", ] } diff --git a/homestar-runtime/tests/cli.rs b/homestar-runtime/tests/cli.rs index aaba16a9..dc0c6c6b 100644 --- a/homestar-runtime/tests/cli.rs +++ b/homestar-runtime/tests/cli.rs @@ -135,14 +135,16 @@ fn test_server_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") + .arg("-c") + .arg("tests/fixtures/test_v6.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) .spawn() .unwrap(); - let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9998); + let result = retry(Fixed::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -153,6 +155,10 @@ fn test_server_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("ping") + .arg("--host") + .arg("::1") + .arg("-p") + .arg("9998") .assert() .success() .stdout(predicate::str::contains("::1")) @@ -160,6 +166,8 @@ fn test_server_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("ping") + .arg("--host") + .arg("::1") .arg("-p") .arg("9999") .assert() @@ -204,7 +212,7 @@ fn test_workflow_run_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(30), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -216,7 +224,7 @@ fn test_workflow_run_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("run") .arg("-w") - .arg("./fixtures/test-workflow-add-one.json") + .arg("tests/fixtures/test-workflow-add-one.json") .assert() .success() .stdout(predicate::str::contains( @@ -232,7 +240,7 @@ fn test_workflow_run_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("run") .arg("-w") - .arg("./fixtures/test-workflow-add-one.json") + .arg("tests/fixtures/test-workflow-add-one.json") .assert() .success() .stdout(predicate::str::contains( @@ -272,14 +280,16 @@ fn test_daemon_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("start") + .arg("-c") + .arg("tests/fixtures/test_v4_alt.toml") .arg("-d") .env("DATABASE_URL", "homestar.db") .stdout(Stdio::piped()) .assert() .success(); - let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9997); + let result = retry(Fixed::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -289,9 +299,13 @@ fn test_daemon_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("ping") + .arg("--host") + .arg("127.0.0.1") + .arg("-p") + .arg("9997") .assert() .success() - .stdout(predicate::str::contains("::1")) + .stdout(predicate::str::contains("127.0.0.1")) .stdout(predicate::str::contains("pong")); let _ = stop_all_bins(); @@ -318,7 +332,7 @@ fn test_signal_kill_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -364,7 +378,7 @@ fn test_server_v4_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") .arg("-c") - .arg("fixtures/test_v4.toml") + .arg("tests/fixtures/test_v4.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) @@ -372,7 +386,7 @@ fn test_server_v4_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(30), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -421,7 +435,7 @@ fn test_daemon_v4_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("start") .arg("-c") - .arg("fixtures/test_v4.toml") + .arg("tests/fixtures/test_v4.toml") .arg("-d") .env("DATABASE_URL", "homestar.db") .stdout(Stdio::piped()) @@ -429,7 +443,7 @@ fn test_daemon_v4_serial() -> Result<()> { .success(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(30), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); diff --git a/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml new file mode 100644 index 00000000..c8f2f4b4 --- /dev/null +++ b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml @@ -0,0 +1,12 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4004 + +[node] + +[node.network] +rpc_port = 3031 +websocket_port = 9090 + +[node.network.keypair_config] +existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" } diff --git a/homestar-runtime/tests/fixtures/network_node1/config/settings.toml b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml new file mode 100644 index 00000000..33090934 --- /dev/null +++ b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml @@ -0,0 +1,16 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4002 + +[node] + +[node.network] +rpc_port = 3032 +websocket_port = 9092 +listen_address = "/ip4/127.0.0.1/tcp/7000" +node_addresses = [ + "/ip4/127.0.0.1/tcp/7001/p2p/16Uiu2HAm3g9AomQNeEctL2hPwLapap7AtPSNt8ZrBny4rLx1W5Dc", +] + +[node.network.keypair_config] +existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" } diff --git a/homestar-runtime/tests/fixtures/network_node2/config/settings.toml b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml new file mode 100644 index 00000000..b1cf1e0a --- /dev/null +++ b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml @@ -0,0 +1,16 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4003 + +[node] + +[node.network] +rpc_port = 3033 +websocket_port = 9093 +listen_address = "/ip4/127.0.0.1/tcp/7001" +node_addresses = [ + "/ip4/127.0.0.1/tcp/7000/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN", +] + +[node.network.keypair_config] +existing = { key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" } diff --git a/homestar-runtime/fixtures/test-workflow-add-one.json b/homestar-runtime/tests/fixtures/test-workflow-add-one.json similarity index 100% rename from homestar-runtime/fixtures/test-workflow-add-one.json rename to homestar-runtime/tests/fixtures/test-workflow-add-one.json diff --git a/homestar-runtime/fixtures/test_v4.toml b/homestar-runtime/tests/fixtures/test_v4.toml similarity index 100% rename from homestar-runtime/fixtures/test_v4.toml rename to homestar-runtime/tests/fixtures/test_v4.toml diff --git a/homestar-runtime/tests/fixtures/test_v4_alt.toml b/homestar-runtime/tests/fixtures/test_v4_alt.toml new file mode 100644 index 00000000..11a3c23f --- /dev/null +++ b/homestar-runtime/tests/fixtures/test_v4_alt.toml @@ -0,0 +1,10 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4007 + +[node] + +[node.network] +events_buffer_len = 1000 +rpc_port = 9997 +rpc_host = "127.0.0.1" diff --git a/homestar-runtime/tests/fixtures/test_v6.toml b/homestar-runtime/tests/fixtures/test_v6.toml new file mode 100644 index 00000000..2e35703f --- /dev/null +++ b/homestar-runtime/tests/fixtures/test_v6.toml @@ -0,0 +1,10 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4006 + +[node] + +[node.network] +events_buffer_len = 1000 +rpc_port = 9998 +rpc_host = "::1" diff --git a/homestar-runtime/tests/metrics.rs b/homestar-runtime/tests/metrics.rs index 7c70209f..889b5dc1 100644 --- a/homestar-runtime/tests/metrics.rs +++ b/homestar-runtime/tests/metrics.rs @@ -5,6 +5,7 @@ use reqwest::StatusCode; use retry::{delay::Fixed, retry, OperationResult}; use serial_test::file_serial; use std::{ + net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream}, path::PathBuf, process::{Command, Stdio}, time::Duration, @@ -12,14 +13,14 @@ use std::{ use wait_timeout::ChildExt; static BIN: Lazy = Lazy::new(|| assert_cmd::cargo::cargo_bin(BIN_NAME)); -const METRICS_URL: &str = "http://localhost:4000"; +const METRICS_URL: &str = "http://localhost:4004"; #[test] #[file_serial] fn test_metrics_serial() -> Result<()> { fn sample_metrics() -> prometheus_parse::Value { let body = retry( - Fixed::from_millis(500).take(2), + Fixed::from_millis(1000).take(10), || match reqwest::blocking::get(METRICS_URL) { Ok(response) => match response.status() { StatusCode::OK => OperationResult::Ok(response.text()), @@ -28,10 +29,9 @@ fn test_metrics_serial() -> Result<()> { Err(_) => OperationResult::Retry("Metrics server not available"), }, ) - .unwrap() - .expect("Metrics server failed to serve metrics"); + .unwrap(); - let lines: Vec<_> = body.lines().map(|s| Ok(s.to_owned())).collect(); + let lines: Vec<_> = body.unwrap().lines().map(|s| Ok(s.to_owned())).collect(); let metrics = prometheus_parse::Scrape::parse(lines.into_iter()).expect("Unable to parse metrics"); @@ -49,16 +49,26 @@ fn test_metrics_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") .arg("-c") - .arg("tests/test_node2/config/settings.toml") + .arg("tests/fixtures/metrics_node/config/settings.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) .spawn() .unwrap(); + let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4004); + let result = retry(Fixed::from_millis(1000).take(10), || { + TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) + }); + + if result.is_err() { + homestar_proc.kill().unwrap(); + panic!("Homestar server/runtime failed to start in time"); + } + let sample1 = sample_metrics(); - let sample2 = retry(Fixed::from_millis(500).take(5), || { + let sample2 = retry(Fixed::from_millis(500).take(3), || { let sample2 = sample_metrics(); if sample1 != sample2 { OperationResult::Ok(sample2) diff --git a/homestar-runtime/tests/test_node1/config/settings.toml b/homestar-runtime/tests/test_node1/config/settings.toml deleted file mode 100644 index 305694c9..00000000 --- a/homestar-runtime/tests/test_node1/config/settings.toml +++ /dev/null @@ -1,13 +0,0 @@ -[monitoring] -process_collector_interval = 500 -metrics_port = 4000 - -[node] - -[node.network] -rpc_port = 3031 -websocket_port = 9091 -node_addresses = ["/ip4/127.0.0.1/tcp/9092/ws"] - -[node.network.keypair_config] -existing = { key_type = "ed25519", path = "../../fixtures/__testkey_ed25519.pem" } diff --git a/homestar-runtime/tests/test_node2/config/settings.toml b/homestar-runtime/tests/test_node2/config/settings.toml deleted file mode 100644 index 92570f77..00000000 --- a/homestar-runtime/tests/test_node2/config/settings.toml +++ /dev/null @@ -1,12 +0,0 @@ -[monitoring] -process_collector_interval = 500 -metrics_port = 4000 - -[node] - -[node.network] -rpc_port = 3032 -websocket_port = 9092 -node_addresses = [ - "/ip4/127.0.0.1/tcp/9091/ws/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN", -] diff --git a/homestar-runtime/tests/utils.rs b/homestar-runtime/tests/utils.rs index 43d94452..afedace7 100644 --- a/homestar-runtime/tests/utils.rs +++ b/homestar-runtime/tests/utils.rs @@ -67,6 +67,7 @@ pub(crate) fn stop_ipfs() -> Result<()> { .stderr(Stdio::null()) .status() .context("failed to stop IPFS daemon")?; + rm_rf::ensure_removed(path).unwrap(); Ok(()) } @@ -97,7 +98,7 @@ pub(crate) fn kill_homestar_process() -> Result<()> { ); let _result = signal::kill(Pid::from_raw(pid.try_into().unwrap()), Signal::SIGTERM); - let _result = retry(Fixed::from_millis(500), || { + let _result = retry(Fixed::from_millis(1000).take(5), || { Command::new(BIN.as_os_str()) .arg("ping") .assert()