feat(agents): improve agent config (#171)

* Make AgentSecrets accept TRANSACTIONSIGNERS_DEFAULT_* envars if present

* Add additional test utils

* Add sync version of run_test_with_env

* Fix bad prefix

* Refactor test to correctly isolate envars

* Add tests for default tx signer config

* Fix linting error

* Add newline to end of file

* Make AgentSecrets accept RPCS_DEFAULT_* envars if present

* Update FromEnv to accept an optional default prefix

* Update test envs with rpc defaults

* Add rpc checks to existing tests

* Bump ethers commit

* Revert "Bump ethers commit"

This reverts commit b262c00.

* Exclude nomad-test from wasm builds

* Use dependency resolver v2

* Add tests for correct defaults handling for kathy

* Update CHANGELOG

* Expand doc for from_env

Cargo.toml

@@ -1,6 +1,7 @@
 cargo-features = ["edition2021"]
 
 [workspace]
+resolver = "2"
 
 members = [
     "accumulator",

agents/kathy/src/settings.rs

@@ -10,7 +10,123 @@ mod test {
     use super::*;
     use nomad_base::{get_remotes_from_env, NomadAgent};
     use nomad_test::test_utils;
-    use nomad_xyz_configuration::AgentSecrets;
+    use nomad_xyz_configuration::{
+        agent::SignerConf, ethereum::Connection, AgentSecrets, ChainConf,
+    };
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn it_builds_settings_from_env_mixed() {
+        test_utils::run_test_with_env("../../fixtures/env.test-signer-mixed", || async move {
+            let run_env = dotenv::var("RUN_ENV").unwrap();
+            let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
+
+            let settings = KathySettings::new().unwrap();
+
+            let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
+
+            let remotes = get_remotes_from_env!(agent_home, config);
+            let mut networks = remotes.clone();
+            networks.insert(agent_home.clone());
+
+            let secrets = AgentSecrets::from_env(&networks).unwrap();
+
+            settings
+                .base
+                .validate_against_config_and_secrets(
+                    crate::Kathy::AGENT_NAME,
+                    &agent_home,
+                    &remotes,
+                    config,
+                    &secrets,
+                )
+                .unwrap();
+
+            assert_eq!(
+                *settings.base.signers.get("moonbeam").unwrap(),
+                SignerConf::Aws {
+                    id: "moonbeam_id".into(),
+                    region: "moonbeam_region".into(),
+                }
+            );
+            assert_eq!(
+                *settings.base.signers.get("ethereum").unwrap(),
+                SignerConf::HexKey(
+                    "0x1111111111111111111111111111111111111111111111111111111111111111"
+                        .parse()
+                        .unwrap()
+                )
+            );
+            assert_eq!(
+                *settings.base.signers.get("evmos").unwrap(),
+                SignerConf::Aws {
+                    id: "default_id".into(),
+                    region: "default_region".into(),
+                }
+            );
+            assert_eq!(
+                settings.base.home.chain,
+                ChainConf::Ethereum(Connection::Http(
+                    "https://main-light.eth.linkpool.io/".into()
+                ))
+            );
+            assert_eq!(
+                settings.base.replicas.get("moonbeam").unwrap().chain,
+                ChainConf::Ethereum(Connection::Http("https://rpc.api.moonbeam.network".into()))
+            );
+            assert_eq!(
+                settings.base.replicas.get("evmos").unwrap().chain,
+                ChainConf::Ethereum(Connection::Http("https://eth.bd.evmos.org:8545".into()))
+            );
+        })
+        .await
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn it_builds_settings_from_env_default() {
+        test_utils::run_test_with_env("../../fixtures/env.test-signer-default", || async move {
+            let run_env = dotenv::var("RUN_ENV").unwrap();
+            let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
+
+            let settings = KathySettings::new().unwrap();
+
+            let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
+
+            let remotes = get_remotes_from_env!(agent_home, config);
+            let mut networks = remotes.clone();
+            networks.insert(agent_home.clone());
+
+            let secrets = AgentSecrets::from_env(&networks).unwrap();
+
+            settings
+                .base
+                .validate_against_config_and_secrets(
+                    crate::Kathy::AGENT_NAME,
+                    &agent_home,
+                    &remotes,
+                    config,
+                    &secrets,
+                )
+                .unwrap();
+
+            let default_config = SignerConf::Aws {
+                id: "default_id".into(),
+                region: "default_region".into(),
+            };
+            for (_, config) in &settings.base.signers {
+                assert_eq!(*config, default_config);
+            }
+            assert!(matches!(
+                settings.base.home.chain,
+                ChainConf::Ethereum { .. }
+            ));
+            for (_, config) in &settings.base.replicas {
+                assert!(matches!(config.chain, ChainConf::Ethereum { .. }));
+            }
+        })
+        .await
+    }
 
     #[tokio::test]
     #[serial_test::serial]

configuration/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Unreleased
 
+- add handling for default keys `TRANSACTIONSIGNERS_DEFAULT_{KEY,ID,REGION}` and `RPCS_DEFAULT_RPCSTYLE`
+- add tests for new default config keys
+
 ### v0.1.0-rc.23
 
 - fix typo in staging goerli rpc url

configuration/Cargo.toml

@@ -35,4 +35,8 @@ js-sys = "0.3.56"
 wasm-bindgen = { version = "0.2.79", features = ["serde-serialize"] }
 
 [dev-dependencies]
-dotenv = "0.15.0"
+dotenv = "0.15.0"
+serial_test = "0.6.0"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+nomad-test = { path = "../nomad-test" }

configuration/src/agent/signer.rs

@@ -28,7 +28,7 @@ impl Default for SignerConf {
 }
 
 impl FromEnv for SignerConf {
-    fn from_env(prefix: &str) -> Option<Self> {
+    fn from_env(prefix: &str, default_prefix: Option<&str>) -> Option<Self> {
         // ordering this first preferentially uses AWS if both are specified
         if let Ok(id) = std::env::var(&format!("{}_ID", prefix)) {
             if let Ok(region) = std::env::var(&format!("{}_REGION", prefix)) {
@@ -40,6 +40,10 @@ impl FromEnv for SignerConf {
             return Some(SignerConf::HexKey(HexString::from_str(&signer_key).ok()?));
         }
 
+        if let Some(prefix) = default_prefix {
+            return SignerConf::from_env(prefix, None);
+        }
+
         None
     }
 }

configuration/src/chains/mod.rs

@@ -23,12 +23,17 @@ impl Default for ChainConf {
 }
 
 impl FromEnv for ChainConf {
-    fn from_env(prefix: &str) -> Option<Self> {
-        let rpc_style = std::env::var(&format!("{}_RPCSTYLE", prefix)).ok()?;
+    fn from_env(prefix: &str, default_prefix: Option<&str>) -> Option<Self> {
+        let mut rpc_style = std::env::var(&format!("{}_RPCSTYLE", prefix)).ok();
+
+        if let (None, Some(prefix)) = (&rpc_style, default_prefix) {
+            rpc_style = std::env::var(&format!("{}_RPCSTYLE", prefix)).ok();
+        }
+
         let rpc_url = std::env::var(&format!("{}_CONNECTION_URL", prefix)).ok()?;
 
         let json = json!({
-            "rpcStyle": rpc_style,
+            "rpcStyle": rpc_style?,
             "connection": rpc_url,
         });
 

configuration/src/secrets.rs

@@ -36,17 +36,22 @@ impl AgentSecrets {
 
         for network in networks.iter() {
             let network_upper = network.to_uppercase();
-            let chain_conf = ChainConf::from_env(&format!("RPCS_{}", network_upper))?;
-            let transaction_signer =
-                SignerConf::from_env(&format!("TRANSACTIONSIGNERS_{}", network_upper))?;
+
+            let chain_conf =
+                ChainConf::from_env(&format!("RPCS_{}", network_upper), Some("RPCS_DEFAULT"))?;
+
+            let transaction_signer = SignerConf::from_env(
+                &format!("TRANSACTIONSIGNERS_{}", network_upper),
+                Some("TRANSACTIONSIGNERS_DEFAULT"),
+            )?;
 
             secrets.rpcs.insert(network.to_owned(), chain_conf);
             secrets
                 .transaction_signers
                 .insert(network.to_owned(), transaction_signer);
         }
 
-        let attestation_signer = SignerConf::from_env("ATTESTATION_SIGNER");
+        let attestation_signer = SignerConf::from_env("ATTESTATION_SIGNER", None);
         secrets.attestation_signer = attestation_signer;
 
         Some(secrets)
@@ -110,18 +115,89 @@ impl AgentSecrets {
 #[cfg(test)]
 mod test {
     use super::*;
-    const SECRETS_JSON_PATH: &str = "../fixtures/test_secrets.json";
-    const SECRETS_ENV_PATH: &str = "../fixtures/env.test";
+    use crate::ethereum::Connection;
+    use nomad_test::test_utils;
+
+    #[test]
+    #[serial_test::serial]
+    fn it_builds_from_env_mixed() {
+        test_utils::run_test_with_env_sync("../fixtures/env.test-signer-mixed", move || {
+            let networks = &crate::get_builtin("test").unwrap().networks;
+            let secrets =
+                AgentSecrets::from_env(networks).expect("Failed to load secrets from env");
+
+            assert_eq!(
+                *secrets.transaction_signers.get("moonbeam").unwrap(),
+                SignerConf::Aws {
+                    id: "moonbeam_id".into(),
+                    region: "moonbeam_region".into(),
+                }
+            );
+            assert_eq!(
+                *secrets.transaction_signers.get("ethereum").unwrap(),
+                SignerConf::HexKey(
+                    "0x1111111111111111111111111111111111111111111111111111111111111111"
+                        .parse()
+                        .unwrap()
+                )
+            );
+            assert_eq!(
+                *secrets.transaction_signers.get("evmos").unwrap(),
+                SignerConf::Aws {
+                    id: "default_id".into(),
+                    region: "default_region".into(),
+                }
+            );
+            assert_eq!(
+                *secrets.rpcs.get("moonbeam").unwrap(),
+                ChainConf::Ethereum(Connection::Http("https://rpc.api.moonbeam.network".into()))
+            );
+            assert_eq!(
+                *secrets.rpcs.get("ethereum").unwrap(),
+                ChainConf::Ethereum(Connection::Http(
+                    "https://main-light.eth.linkpool.io/".into()
+                ))
+            );
+            assert_eq!(
+                *secrets.rpcs.get("evmos").unwrap(),
+                ChainConf::Ethereum(Connection::Http("https://eth.bd.evmos.org:8545".into()))
+            );
+        });
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn it_builds_from_env_default() {
+        test_utils::run_test_with_env_sync("../fixtures/env.test-signer-default", move || {
+            let networks = &crate::get_builtin("test").unwrap().networks;
+            let secrets =
+                AgentSecrets::from_env(networks).expect("Failed to load secrets from env");
+
+            let default_config = SignerConf::Aws {
+                id: "default_id".into(),
+                region: "default_region".into(),
+            };
+            for (_, config) in &secrets.transaction_signers {
+                assert_eq!(*config, default_config);
+            }
+            for (_, config) in &secrets.rpcs {
+                assert!(matches!(*config, ChainConf::Ethereum { .. }));
+            }
+        });
+    }
 
     #[test]
+    #[serial_test::serial]
     fn it_builds_from_env() {
-        let networks = &crate::get_builtin("test").unwrap().networks;
-        dotenv::from_filename(SECRETS_ENV_PATH).unwrap();
-        AgentSecrets::from_env(networks).expect("Failed to load secrets from env");
+        test_utils::run_test_with_env_sync("../fixtures/env.test", move || {
+            let networks = &crate::get_builtin("test").unwrap().networks;
+            AgentSecrets::from_env(networks).expect("Failed to load secrets from env");
+        });
     }
 
     #[test]
     fn it_builds_from_file() {
-        AgentSecrets::from_file(SECRETS_JSON_PATH).expect("Failed to load secrets from file");
+        AgentSecrets::from_file("../fixtures/test_secrets.json")
+            .expect("Failed to load secrets from file");
     }
 }

configuration/src/traits/env.rs

@@ -7,9 +7,11 @@ pub trait EnvOverridable {
 /// Implemented by structs that are built from environment variables (signers,
 /// connections, etc)
 pub trait FromEnv {
-    /// Optionally load self from env vars. Return None if any necessary env var
-    /// is missing.
-    fn from_env(prefix: &str) -> Option<Self>
+    /// Optionally load self from env vars.
+    /// Accepts a `default_prefix` which will be looked for if `prefix` isn't found.
+    /// If both are present, `prefix` has precedence over `default_prefix`.
+    /// Return None if *any* necessary env var is missing.
+    fn from_env(prefix: &str, default_prefix: Option<&str>) -> Option<Self>
     where
         Self: Sized;
 }

fixtures/env.test-signer-default

@@ -0,0 +1,17 @@
+# Matches configuration/configs/test.json
+
+RUN_ENV=test
+AGENT_HOME_NAME=ethereum
+AGENT_REPLICAS_ALL=true
+
+RPCS_DEFAULT_RPCSTYLE=ethereum
+
+RPCS_ETHEREUM_CONNECTION_URL=https://main-light.eth.linkpool.io/
+RPCS_MOONBEAM_CONNECTION_URL=https://rpc.api.moonbeam.network
+RPCS_EVMOS_CONNECTION_URL=https://eth.bd.evmos.org:8545
+
+TRANSACTIONSIGNERS_DEFAULT_REGION=default_region
+TRANSACTIONSIGNERS_DEFAULT_ID=default_id
+
+ATTESTATION_SIGNER_ID=dummy_id
+ATTESTATION_SIGNER_REGION=dummy_region

fixtures/env.test-signer-mixed

@@ -0,0 +1,23 @@
+# Matches configuration/configs/test.json
+
+RUN_ENV=test
+AGENT_HOME_NAME=ethereum
+AGENT_REPLICAS_ALL=true
+
+RPCS_DEFAULT_RPCSTYLE=ethereum
+RPCS_EVMOS_RPCSTYLE=ethereum
+
+RPCS_ETHEREUM_CONNECTION_URL=https://main-light.eth.linkpool.io/
+RPCS_MOONBEAM_CONNECTION_URL=https://rpc.api.moonbeam.network
+RPCS_EVMOS_CONNECTION_URL=https://eth.bd.evmos.org:8545
+
+TRANSACTIONSIGNERS_ETHEREUM_KEY=0x1111111111111111111111111111111111111111111111111111111111111111
+
+TRANSACTIONSIGNERS_MOONBEAM_ID=moonbeam_id
+TRANSACTIONSIGNERS_MOONBEAM_REGION=moonbeam_region
+
+TRANSACTIONSIGNERS_DEFAULT_REGION=default_region
+TRANSACTIONSIGNERS_DEFAULT_ID=default_id
+
+ATTESTATION_SIGNER_ID=dummy_id
+ATTESTATION_SIGNER_REGION=dummy_region