chore: updates/flaky kills on ci (#540)

Includes:

- Handling only the unstable packages we need in nix
- Cleaner daemon test for a retry as GH-actions will kill it from time
to time.
- return threads-required for dht workflow test

.config/nextest.toml

@@ -21,6 +21,7 @@ test-group = 'serial'
 
 [[profile.default.overrides]]
 filter = 'test(/_integration$/)'
+threads-required = 3
 test-group = 'integration'
 
 [[profile.ci.overrides]]
@@ -29,4 +30,5 @@ test-group = 'serial'
 
 [[profile.ci.overrides]]
 filter = 'test(/_integration$/)'
+threads-required = 3
 test-group = 'integration'

.github/workflows/dependabot_pr.yaml

@@ -1,4 +1,4 @@
-name: Dependabot PR
+name: 🤖 Dependabot PR
 
 on:
   pull_request_target:

flake.nix

@@ -2,8 +2,9 @@
   description = "homestar";
 
   inputs = {
+    nixpkgs.url = "nixpkgs/nixos-23.11";
     # we leverage unstable due to wasm-tools/wasm updates
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixos-unstable.url = "nixpkgs/nixos-unstable-small";
     flake-utils.url = "github:numtide/flake-utils";
     nixlib.url = "github:nix-community/nixpkgs.lib";
 
@@ -39,6 +40,7 @@
   outputs = {
     self,
     nixpkgs,
+    nixos-unstable,
     fenix,
     flake-compat,
     flake-utils,
@@ -50,6 +52,7 @@
       system: let
         overlays = [(import rust-overlay) fenix.overlays.default wit-deps.overlays.default];
         pkgs = import nixpkgs {inherit system overlays;};
+        unstable = import nixos-unstable {inherit system overlays;};
 
         rust-toolchain = (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml).override {
           extensions = ["cargo" "clippy" "llvm-tools-preview" "rustfmt" "rust-src" "rust-std"];
@@ -74,7 +77,7 @@
 
         cargo-installs = with pkgs; [
           cargo-deb
-          # cargo-deny -> bring back on new release
+          cargo-deny
           cargo-cross
           cargo-expand
           cargo-hakari
@@ -86,7 +89,8 @@
           rustup
           tokio-console
           twiggy
-          wasm-tools
+          unstable.cargo-component
+          unstable.wasm-tools
         ];
 
         ci = pkgs.writeScriptBin "ci" ''
@@ -282,7 +286,8 @@
               pre-commit
               diesel-cli
               direnv
-              nodejs_18
+              unstable.nodejs_20
+              unstable.nodePackages.pnpm
               kubo
               self.packages.${system}.irust
             ]

homestar-runtime/tests/cli.rs

@@ -236,8 +236,9 @@ fn test_workflow_run_integration() -> Result<()> {
 }
 
 #[test]
+#[serial_test::file_serial]
 #[cfg(not(windows))]
-fn test_daemon_integration() -> Result<()> {
+fn test_daemon_serial() -> Result<()> {
     let proc_info = ProcInfo::new().unwrap();
     let rpc_port = proc_info.rpc_port;
     let metrics_port = proc_info.metrics_port;
@@ -269,20 +270,29 @@ fn test_daemon_integration() -> Result<()> {
         .success();
 
     if wait_for_socket_connection(rpc_port, 1000).is_err() {
-        let _ = kill_homestar_daemon();
+        kill_homestar_daemon().unwrap();
         panic!("Homestar server/runtime failed to start in time");
     }
 
-    Command::new(BIN.as_os_str())
+    let res = Command::new(BIN.as_os_str())
         .arg("ping")
         .arg("--host")
         .arg("127.0.0.1")
         .arg("-p")
         .arg(rpc_port.to_string())
         .assert()
-        .success()
-        .stdout(predicate::str::contains("127.0.0.1"))
-        .stdout(predicate::str::contains("pong"));
+        .try_success();
+
+    match res {
+        Ok(ok) => {
+            ok.stdout(predicate::str::contains("127.0.0.1"))
+                .stdout(predicate::str::contains("pong"));
+        }
+        Err(err) => {
+            kill_homestar_daemon().unwrap();
+            panic!("Err: {:?}", err);
+        }
+    }
 
     kill_homestar_daemon().unwrap();
     Ok(())

homestar-runtime/tests/network/dht.rs

@@ -33,7 +33,7 @@ const UNSUBSCRIBE_NETWORK_EVENTS_ENDPOINT: &str = "unsubscribe_network_events";
 
 #[test]
 #[serial_test::file_serial]
-fn test_libp2p_dht_records_serial() -> Result<()> {
+fn test_libp2p_dht_records_integration() -> Result<()> {
     let proc_info1 = ProcInfo::new().unwrap();
     let proc_info2 = ProcInfo::new().unwrap();
 
@@ -367,7 +367,7 @@ fn test_libp2p_dht_records_serial() -> Result<()> {
 
 #[test]
 #[serial_test::file_serial]
-fn test_libp2p_dht_quorum_failure_serial() -> Result<()> {
+fn test_libp2p_dht_quorum_failure_intregration() -> Result<()> {
     let proc_info1 = ProcInfo::new().unwrap();
     let proc_info2 = ProcInfo::new().unwrap();
 
@@ -566,7 +566,7 @@ fn test_libp2p_dht_quorum_failure_serial() -> Result<()> {
 
 #[test]
 #[serial_test::file_serial]
-fn test_libp2p_dht_workflow_info_provider_serial() -> Result<()> {
+fn test_libp2p_dht_workflow_info_provider_integration() -> Result<()> {
     let proc_info1 = ProcInfo::new().unwrap();
     let proc_info2 = ProcInfo::new().unwrap();
 
@@ -588,7 +588,6 @@ fn test_libp2p_dht_workflow_info_provider_serial() -> Result<()> {
         [node.network.libp2p]
         listen_address = "{listen_addr1}"
         node_addresses = ["{node_addrb}"]
-        idle_connection_timeout = 180
         [node.network.libp2p.dht]
         receipt_quorum = 1
         workflow_quorum = 1
@@ -649,7 +648,6 @@ fn test_libp2p_dht_workflow_info_provider_serial() -> Result<()> {
         [node.network.keypair_config]
         existing = {{ key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" }}
         [node.network.libp2p]
-        idle_connection_timeout = 180
         listen_address = "{listen_addr2}"
         node_addresses = ["{node_addra}"]
         [node.network.libp2p.dht]
@@ -732,7 +730,7 @@ fn test_libp2p_dht_workflow_info_provider_serial() -> Result<()> {
         // We want node two to request workflow info directly from node one
         // because of timeouts not because workflow info was missing from the
         // DHT, so we give node one time to put add workflow info to the DHT.
-        tokio::time::sleep(Duration::from_secs(7)).await;
+        tokio::time::sleep(Duration::from_secs(9)).await;
 
         // Run the same workflow run on node two.
         // Node two should be request workflow info from
@@ -749,7 +747,7 @@ fn test_libp2p_dht_workflow_info_provider_serial() -> Result<()> {
         // Poll for sent workflow info message
         let sent_workflow_info_cid: Cid;
         loop {
-            if let Ok(msg) = sub1.next().with_timeout(Duration::from_secs(30)).await {
+            if let Ok(msg) = sub1.next().with_timeout(Duration::from_secs(60)).await {
                 let json: serde_json::Value =
                     serde_json::from_slice(&msg.unwrap().unwrap()).unwrap();
 
@@ -859,7 +857,7 @@ fn test_libp2p_dht_workflow_info_provider_serial() -> Result<()> {
 #[ignore]
 #[test]
 #[serial_test::file_serial]
-fn test_libp2p_dht_workflow_info_provider_recursive_serial() -> Result<()> {
+fn test_libp2p_dht_workflow_info_provider_recursive_integration() -> Result<()> {
     // NOTE: We are ignoring this test for now because we do not have a means
     // to properly isolate node a from node c. In the future when nodes are
     // partitioned as private nodes or from NATs, we will bring this test back.