From 5cb673ef443a41aa7d2b6a834d912840f256f656 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Mon, 30 Sep 2024 13:02:27 -0500 Subject: [PATCH 1/2] Teach upgrade checker to use aws-secrets-reader --- Cargo.lock | 2 ++ src/catalog-debug/BUILD.bazel | 3 +- src/catalog-debug/Cargo.toml | 2 ++ src/catalog-debug/src/main.rs | 59 ++++++++++++++++++++++++++++------- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77aa4df3e789..79102bb2aced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4471,11 +4471,13 @@ dependencies = [ "mz-adapter", "mz-build-info", "mz-catalog", + "mz-cloud-resources", "mz-orchestrator-tracing", "mz-ore", "mz-persist-client", "mz-repr", "mz-secrets", + "mz-service", "mz-sql", "mz-storage-types", "mz-tls-util", diff --git a/src/catalog-debug/BUILD.bazel b/src/catalog-debug/BUILD.bazel index 42376ab8c7b0..70455dee5e4b 100644 --- a/src/catalog-debug/BUILD.bazel +++ b/src/catalog-debug/BUILD.bazel @@ -34,11 +34,12 @@ rust_binary( "//src/adapter:mz_adapter", "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", + "//src/cloud-resources:mz_cloud_resources", "//src/orchestrator-tracing:mz_orchestrator_tracing", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/repr:mz_repr", - "//src/secrets:mz_secrets", + "//src/service:mz_service", "//src/sql:mz_sql", "//src/storage-types:mz_storage_types", "//src/tls-util:mz_tls_util", diff --git a/src/catalog-debug/Cargo.toml b/src/catalog-debug/Cargo.toml index 1745b0ffc0e0..ca9706a26f6e 100644 --- a/src/catalog-debug/Cargo.toml +++ b/src/catalog-debug/Cargo.toml @@ -16,6 +16,7 @@ futures = "0.3.25" mz-adapter = { path = "../adapter" } mz-build-info = { path = "../build-info" } mz-catalog = { path = "../catalog" } +mz-cloud-resources = { path = "../cloud-resources" } mz-orchestrator-tracing = { path = "../orchestrator-tracing" } mz-ore = { path = "../ore" } mz-storage-types = { path = "../storage-types" } @@ -23,6 +24,7 @@ mz-persist-client = { path = "../persist-client" } mz-tls-util = { path = "../tls-util" } mz-repr = { path = "../repr" } mz-secrets = { path = "../secrets" } +mz-service = { path = "../service" } mz-sql = { path = "../sql" } serde = "1.0.152" serde_json = "1.0.125" diff --git a/src/catalog-debug/src/main.rs b/src/catalog-debug/src/main.rs index e5feb48ef75f..8566b9e38899 100644 --- a/src/catalog-debug/src/main.rs +++ b/src/catalog-debug/src/main.rs @@ -37,6 +37,7 @@ use mz_catalog::durable::debug::{ use mz_catalog::durable::{ persist_backed_catalog_state, BootstrapArgs, OpenableDurableCatalogState, }; +use mz_cloud_resources::AwsExternalIdPrefix; use mz_orchestrator_tracing::{StaticTracingConfig, TracingCliArgs}; use mz_ore::cli::{self, CliConfig}; use mz_ore::collections::HashSet; @@ -48,7 +49,7 @@ use mz_persist_client::cfg::PersistConfig; use mz_persist_client::rpc::PubSubClientConnection; use mz_persist_client::PersistLocation; use mz_repr::{Diff, Timestamp}; -use mz_secrets::InMemorySecretsController; +use mz_service::secrets::SecretsReaderCliArgs; use mz_sql::catalog::EnvironmentId; use mz_sql::session::vars::ConnectionCounter; use mz_storage_types::connections::ConnectionContext; @@ -73,15 +74,37 @@ pub struct Args { /// Where the persist library should perform consensus. #[clap(long, env = "PERSIST_CONSENSUS_URL")] persist_consensus_url: Url, + // === Cloud options. === + /// An external ID to be supplied to all AWS AssumeRole operations. + /// + /// Details: + #[clap(long, env = "AWS_EXTERNAL_ID", value_name = "ID", parse(from_str = AwsExternalIdPrefix::new_from_cli_argument_or_environment_variable))] + aws_external_id_prefix: Option, + + /// The ARN for a Materialize-controlled role to assume before assuming + /// a customer's requested role for an AWS connection. + #[clap(long, env = "AWS_CONNECTION_ROLE_ARN")] + aws_connection_role_arn: Option, + // === Secrets reader options. === + #[clap(flatten)] + secrets: SecretsReaderCliArgs, + // === Tracing options. === + #[clap(flatten)] + tracing: TracingCliArgs, + // === Other options. === + /// An opaque identifier for the environment in which this process is + /// running. + #[clap(long, env = "ENVIRONMENT_ID")] + environment_id: EnvironmentId, + + #[clap(long)] + deploy_generation: Option, #[clap(subcommand)] action: Action, - - #[clap(flatten)] - tracing: TracingCliArgs, } -#[derive(Debug, clap::Subcommand)] +#[derive(Debug, Clone, clap::Subcommand)] enum Action { /// Dumps the catalog contents to stdout in a human-readable format. /// Includes JSON for each key and value that can be hand edited and @@ -185,12 +208,12 @@ async fn run(args: Args) -> Result<(), anyhow::Error> { persist_client, organization_id, BUILD_INFO.semver_version(), - None, + args.deploy_generation, metrics, ) .await?; - match args.action { + match args.action.clone() { Action::Dump { ignore_large_collections, ignore, @@ -235,7 +258,7 @@ async fn run(args: Args) -> Result<(), anyhow::Error> { None => Default::default(), Some(json) => serde_json::from_str(&json).context("parsing replica size map")?, }; - upgrade_check(openable_state, cluster_replica_sizes, start).await + upgrade_check(args, openable_state, cluster_replica_sizes, start).await } } } @@ -498,10 +521,17 @@ async fn epoch( } async fn upgrade_check( + args: Args, openable_state: Box, cluster_replica_sizes: ClusterReplicaSizeMap, start: Instant, ) -> Result<(), anyhow::Error> { + let secrets_reader = args + .secrets + .load() + .await + .context("loading secrets reader")?; + let now = SYSTEM_TIME.clone(); let mut storage = openable_state .open_savepoint( @@ -539,7 +569,7 @@ async fn upgrade_check( unsafe_mode: true, all_features: false, build_info: &BUILD_INFO, - environment_id: EnvironmentId::for_tests(), + environment_id: args.environment_id.clone(), now, boot_ts, skip_migrations: false, @@ -556,9 +586,14 @@ async fn upgrade_check( aws_principal_context: None, aws_privatelink_availability_zones: None, http_host_name: None, - connection_context: ConnectionContext::for_tests(Arc::new( - InMemorySecretsController::new(), - )), + connection_context: ConnectionContext::from_cli_args( + args.environment_id.to_string(), + &args.tracing.startup_log_filter, + args.aws_external_id_prefix, + args.aws_connection_role_arn, + secrets_reader, + None, + ), active_connection_count: Arc::new(Mutex::new(ConnectionCounter::new(0, 0))), builtin_item_migration_config: BuiltinItemMigrationConfig::Legacy, }, From 912ad7848776b18fcc3ba5dfcec31d7f0665ee4d Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Mon, 30 Sep 2024 21:57:05 -0500 Subject: [PATCH 2/2] Revert "catalog: Remove deploy gen req from savepoint (#29765)" This reverts commit 76bdf33866ed305e7a1576c689f8cfb4f169a955. --- Cargo.lock | 1 - src/catalog-debug/Cargo.toml | 1 - src/catalog/src/durable/persist.rs | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79102bb2aced..8338c10ba832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4476,7 +4476,6 @@ dependencies = [ "mz-ore", "mz-persist-client", "mz-repr", - "mz-secrets", "mz-service", "mz-sql", "mz-storage-types", diff --git a/src/catalog-debug/Cargo.toml b/src/catalog-debug/Cargo.toml index ca9706a26f6e..8bd7a5ea5a23 100644 --- a/src/catalog-debug/Cargo.toml +++ b/src/catalog-debug/Cargo.toml @@ -23,7 +23,6 @@ mz-storage-types = { path = "../storage-types" } mz-persist-client = { path = "../persist-client" } mz-tls-util = { path = "../tls-util" } mz-repr = { path = "../repr" } -mz-secrets = { path = "../secrets" } mz-service = { path = "../service" } mz-sql = { path = "../sql" } serde = "1.0.152" diff --git a/src/catalog/src/durable/persist.rs b/src/catalog/src/durable/persist.rs index 682de5a2ccc5..3aa77b874e42 100644 --- a/src/catalog/src/durable/persist.rs +++ b/src/catalog/src/durable/persist.rs @@ -1088,7 +1088,7 @@ impl UnopenedPersistCatalogState { .into()); } ( - Mode::Writable, + Mode::Writable | Mode::Savepoint, FenceableToken::Initializing { current_deploy_generation: None, ..