feat: add generic config list type and use throughout all config sett…

…ings (#6567)

Description
---
Added a generic config list type for use with config files and applied
this to config arrays of type 'Vec<T>'. With this PR, all array-type
config options can be omitted in the config file and/or passed in via
the command line as a config property override with `-p <override>'.

Motivation and Context
---
Using config options where arrays have been used was not flexible or
consistent throughout the config implementation.

How Has This Been Tested?
---
Added unit tests.
System-level testing.

What process can a PR reviewer use to test or verify this change?
---
Code review.
System-level testing.

<!-- Checklist -->
<!-- 1. Is the title of your PR in the form that would make nice release
notes? The title, excluding the conventional commit
tag, will be included exactly as is in the CHANGELOG, so please think
about it carefully. -->


Breaking Changes
---

- [x] None
- [ ] Requires data directory on base node to be deleted
- [ ] Requires hard fork
- [ ] Other - Please specify

<!-- Does this include a breaking change? If so, include this line as a
footer -->
<!-- BREAKING CHANGE: Description what the user should do, e.g. delete a
database, resync the chain -->

applications/minotari_app_utilities/src/common_cli_args.rs

@@ -50,7 +50,12 @@ pub struct CommonCliArgs {
     #[clap(long, env = "TARI_NETWORK")]
     pub network: Option<Network>,
 
-    /// Overrides for properties in the config file, e.g. -p base_node.network=esmeralda
+    /// Overrides for properties in the config file (use the fully qualified key name!!), e.g.
+    /// -p base_node.network=esmeralda
+    /// -p base_node.grpc_server_allow_methods="get_tokens_in_circulation, get_sync_progress, get_mempool_stats"
+    /// -p esmeralda.p2p.seeds.peer_seeds="<public_key_1>::<address_1>", "<public_key_2>::<address_2>", or,
+    /// -p base_node.grpc_server_allow_methods=""
+    /// -p esmeralda.p2p.seeds.peer_seeds=""
     #[clap(short = 'p', parse(try_from_str = parse_key_val), multiple_occurrences(true))]
     pub config_property_overrides: Vec<(String, String)>,
 }
@@ -105,7 +110,7 @@ impl CommonCliArgs {
 }
 
 impl ConfigOverrideProvider for CommonCliArgs {
-    fn get_config_property_overrides(&self, _network: &mut Network) -> Vec<(String, String)> {
+    fn get_config_property_overrides(&self, _network: &Network) -> Vec<(String, String)> {
         let mut overrides = self.config_property_overrides.clone();
         overrides.push((
             "common.base_path".to_string(),

applications/minotari_console_wallet/src/cli.rs

@@ -96,25 +96,38 @@ pub struct Cli {
 }
 
 impl ConfigOverrideProvider for Cli {
-    fn get_config_property_overrides(&self, network: &mut Network) -> Vec<(String, String)> {
-        let mut overrides = self.common.get_config_property_overrides(network);
-        *network = self.common.network.unwrap_or(*network);
-        overrides.push(("wallet.network".to_string(), network.to_string()));
+    /// Get the configuration property overrides for the given network. In case of duplicates, the final override
+    /// added to the list will have preference.
+    fn get_config_property_overrides(&self, network: &Network) -> Vec<(String, String)> {
+        // Config file overrides
+        let mut overrides = vec![("wallet.network".to_string(), network.to_string())];
         overrides.push(("wallet.override_from".to_string(), network.to_string()));
         overrides.push(("p2p.seeds.override_from".to_string(), network.to_string()));
-        // Either of these configs enable grpc
+        // Command-line overrides
+        let command_line_overrides = self.common.get_config_property_overrides(network);
+        command_line_overrides.iter().for_each(|(k, v)| {
+            replace_or_add_override(&mut overrides, k, v);
+        });
+        // Logical overrides based on command-line flags - Either of these configs enable grpc
         if let Some(ref addr) = self.grpc_address {
-            overrides.push(("wallet.grpc_enabled".to_string(), "true".to_string()));
-            overrides.push(("wallet.grpc_address".to_string(), addr.clone()));
+            replace_or_add_override(&mut overrides, "wallet.grpc_enabled", "true");
+            replace_or_add_override(&mut overrides, "wallet.grpc_address", addr);
         } else if self.grpc_enabled {
-            overrides.push(("wallet.grpc_enabled".to_string(), "true".to_string()));
+            replace_or_add_override(&mut overrides, "wallet.grpc_enabled", "true");
         } else {
             // GRPC is disabled
         }
         overrides
     }
 }
 
+fn replace_or_add_override(overrides: &mut Vec<(String, String)>, key: &str, value: &str) {
+    if let Some(index) = overrides.iter().position(|(k, _)| k == key) {
+        overrides.remove(index);
+    }
+    overrides.push((key.to_string(), value.to_string()));
+}
+
 #[allow(clippy::large_enum_variant)]
 #[derive(Debug, Subcommand, Clone)]
 pub enum CliCommands {

applications/minotari_console_wallet/src/main.rs

@@ -24,6 +24,7 @@ use std::process;
 
 use clap::Parser;
 use log::*;
+use minotari_app_utilities::consts;
 use minotari_console_wallet::{run_wallet_with_cli, ApplicationConfig, Cli};
 use tari_common::{
     configuration::bootstrap::{grpc_default_port, ApplicationType},
@@ -69,14 +70,25 @@ fn main() {
 
 fn main_inner() -> Result<(), ExitError> {
     let cli = Cli::parse();
-
-    let cfg = load_configuration(cli.common.config_path(), true, cli.non_interactive_mode, &cli)?;
     let base_path = cli.common.get_base_path();
     initialize_logging(
         &cli.common.log_config_path("wallet"),
         cli.common.log_path.as_ref().unwrap_or(&base_path),
         include_str!("../log4rs_sample.yml"),
     )?;
+    info!(
+        target: LOG_TARGET,
+        "Starting Minotari Console Wallet version: {}",
+        consts::APP_VERSION
+    );
+
+    let cfg = load_configuration(
+        cli.common.config_path(),
+        true,
+        cli.non_interactive_mode,
+        &cli,
+        cli.common.network,
+    )?;
 
     if cli.profile_with_tokio_console {
         // Uncomment to enable tokio tracing via tokio-console

applications/minotari_merge_mining_proxy/src/cli.rs

@@ -35,11 +35,24 @@ pub struct Cli {
 }
 
 impl ConfigOverrideProvider for Cli {
-    fn get_config_property_overrides(&self, network: &mut Network) -> Vec<(String, String)> {
-        let mut overrides = self.common.get_config_property_overrides(network);
-        *network = self.common.network.unwrap_or(*network);
-        overrides.push(("merge_mining_proxy.override_from".to_string(), network.to_string()));
+    /// Get the configuration property overrides for the given network. In case of duplicates, the final override
+    /// added to the list will have preference.
+    fn get_config_property_overrides(&self, network: &Network) -> Vec<(String, String)> {
+        // Config file overrides
+        let mut overrides = vec![("merge_mining_proxy.override_from".to_string(), network.to_string())];
         overrides.push(("merge_mining_proxy.network".to_string(), network.to_string()));
+        // Command-line overrides
+        let command_line_overrides = self.common.get_config_property_overrides(network);
+        command_line_overrides.iter().for_each(|(k, v)| {
+            replace_or_add_override(&mut overrides, k, v);
+        });
         overrides
     }
 }
+
+fn replace_or_add_override(overrides: &mut Vec<(String, String)>, key: &str, value: &str) {
+    if let Some(index) = overrides.iter().position(|(k, _)| k == key) {
+        overrides.remove(index);
+    }
+    overrides.push((key.to_string(), value.to_string()));
+}

applications/minotari_merge_mining_proxy/src/main.rs

@@ -59,6 +59,11 @@ async fn main() -> Result<(), anyhow::Error> {
         cli.common.log_path.as_ref().unwrap_or(&base_path),
         include_str!("../log4rs_sample.yml"),
     )?;
+    info!(
+        target: LOG_TARGET,
+        "Starting Minotari Merge Mining Proxy version: {}",
+        consts::APP_VERSION
+    );
     match run_merge_miner::start_merge_miner(cli).await {
         Ok(_) => Ok(()),
         Err(err) => {

applications/minotari_merge_mining_proxy/src/run_merge_miner.rs

@@ -54,7 +54,7 @@ const LOG_TARGET: &str = "minotari_mm_proxy::proxy";
 
 pub async fn start_merge_miner(cli: Cli) -> Result<(), anyhow::Error> {
     let config_path = cli.common.config_path();
-    let cfg = load_configuration(&config_path, true, cli.non_interactive_mode, &cli)?;
+    let cfg = load_configuration(&config_path, true, cli.non_interactive_mode, &cli, cli.common.network)?;
     let mut config = MergeMiningProxyConfig::load_from(&cfg)?;
     config.set_base_path(cli.common.get_base_path());
     if config.use_dynamic_fail_data {

applications/minotari_miner/src/cli.rs

@@ -42,10 +42,23 @@ pub struct Cli {
 }
 
 impl ConfigOverrideProvider for Cli {
-    fn get_config_property_overrides(&self, network: &mut Network) -> Vec<(String, String)> {
-        let mut overrides = self.common.get_config_property_overrides(network);
-        *network = self.common.network.unwrap_or(*network);
-        overrides.push(("miner.network".to_string(), network.to_string()));
+    /// Get the configuration property overrides for the given network. In case of duplicates, the final override
+    /// added to the list will have preference.
+    fn get_config_property_overrides(&self, network: &Network) -> Vec<(String, String)> {
+        // Config file overrides
+        let mut overrides = vec![("miner.network".to_string(), network.to_string())];
+        // Command-line overrides
+        let command_line_overrides = self.common.get_config_property_overrides(network);
+        command_line_overrides.iter().for_each(|(k, v)| {
+            replace_or_add_override(&mut overrides, k, v);
+        });
         overrides
     }
 }
+
+fn replace_or_add_override(overrides: &mut Vec<(String, String)>, key: &str, value: &str) {
+    if let Some(index) = overrides.iter().position(|(k, _)| k == key) {
+        overrides.remove(index);
+    }
+    overrides.push((key.to_string(), value.to_string()));
+}

applications/minotari_miner/src/main.rs

@@ -67,5 +67,10 @@ async fn main_inner() -> Result<(), ExitError> {
         &cli.common.get_base_path(),
         include_str!("../log4rs_sample.yml"),
     )?;
+    info!(
+        target: LOG_TARGET,
+        "Starting Minotari Miner version: {}",
+        consts::APP_VERSION
+    );
     start_miner(cli).await
 }

applications/minotari_miner/src/run_miner.rs

@@ -81,7 +81,13 @@ pub const LOG_TARGET_FILE: &str = "minotari::logging::miner::main";
 #[allow(clippy::too_many_lines)]
 pub async fn start_miner(cli: Cli) -> Result<(), ExitError> {
     let config_path = cli.common.config_path();
-    let cfg = load_configuration(config_path.as_path(), true, cli.non_interactive_mode, &cli)?;
+    let cfg = load_configuration(
+        config_path.as_path(),
+        true,
+        cli.non_interactive_mode,
+        &cli,
+        cli.common.network,
+    )?;
     let mut config = MinerConfig::load_from(&cfg).expect("Failed to load config");
     config.set_base_path(cli.common.get_base_path());
 

applications/minotari_node/Cargo.toml

@@ -45,7 +45,8 @@ log4rs = { version = "1.3.0", default-features = false, features = ["config_pars
 nom = "7.1"
 rustyline = "9.0"
 rustyline-derive = "0.5"
-serde = "1.0.136"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 strum = { version = "0.22", features = ["derive"] }
 thiserror = "^1.0.26"
 tokio = { version = "1.36", features = ["signal"] }
@@ -71,3 +72,4 @@ ignored = [
 
 [dev-dependencies]
 toml = { version = "0.5" }
+serde_json = "1.0.108"

applications/minotari_node/src/cli.rs

@@ -54,25 +54,39 @@ pub struct Cli {
 }
 
 impl ConfigOverrideProvider for Cli {
-    fn get_config_property_overrides(&self, network: &mut Network) -> Vec<(String, String)> {
-        let mut overrides = self.common.get_config_property_overrides(network);
-        *network = self.common.network.unwrap_or(*network);
-        overrides.push(("base_node.network".to_string(), network.to_string()));
+    /// Get the configuration property overrides for the given network. In case of duplicates, the final override
+    /// added to the list will have preference.
+    fn get_config_property_overrides(&self, network: &Network) -> Vec<(String, String)> {
+        // Config file overrides
+        let mut overrides = vec![("base_node.network".to_string(), network.to_string())];
         overrides.push(("base_node.override_from".to_string(), network.to_string()));
         overrides.push(("p2p.seeds.override_from".to_string(), network.to_string()));
         overrides.push(("auto_update.override_from".to_string(), network.to_string()));
         overrides.push(("metrics.override_from".to_string(), network.to_string()));
+        // Command-line overrides
+        let command_line_overrides = self.common.get_config_property_overrides(network);
+        command_line_overrides.iter().for_each(|(k, v)| {
+            replace_or_add_override(&mut overrides, k, v);
+        });
+        // Logical overrides based on command-line flags
         if self.grpc_enabled {
-            overrides.push(("base_node.grpc_enabled".to_string(), "true".to_string()));
+            replace_or_add_override(&mut overrides, "base_node.grpc_enabled", "true");
         }
         if self.mining_enabled {
-            overrides.push(("base_node.grpc_enabled".to_string(), "true".to_string()));
-            overrides.push(("base_node.mining_enabled".to_string(), "true".to_string()));
+            replace_or_add_override(&mut overrides, "base_node.grpc_enabled", "true");
+            replace_or_add_override(&mut overrides, "base_node.mining_enabled", "true");
         }
         if self.second_layer_grpc_enabled {
-            overrides.push(("base_node.grpc_enabled".to_string(), "true".to_string()));
-            overrides.push(("base_node.second_layer_grpc_enabled".to_string(), "true".to_string()));
+            replace_or_add_override(&mut overrides, "base_node.grpc_enabled", "true");
+            replace_or_add_override(&mut overrides, "base_node.second_layer_grpc_enabled", "true");
         }
         overrides
     }
 }
+
+fn replace_or_add_override(overrides: &mut Vec<(String, String)>, key: &str, value: &str) {
+    if let Some(index) = overrides.iter().position(|(k, _)| k == key) {
+        overrides.remove(index);
+    }
+    overrides.push((key.to_string(), value.to_string()));
+}