From a2c79412027f3f4a866f3ac61f6965d321bebdcc Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 2 Nov 2023 09:50:43 +0100 Subject: [PATCH] Use updated product-config --- Cargo.lock | 184 +++++++++++++++++++++++-- Cargo.toml | 9 +- rust/crd/src/lib.rs | 8 +- rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/controller.rs | 73 +++++++--- 5 files changed, 237 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb4ef91c..0e75d391 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,6 +195,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "built" version = "0.6.1" @@ -347,6 +356,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -366,6 +384,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.14.4" @@ -447,12 +475,36 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dockerfile-parser" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75799314f5fa405629a365a1f97d80f81edd17f22a0fc9c8ddb3ad191ad8dc08" +dependencies = [ + "enquote", + "lazy_static", + "pest", + "pest_derive", + "regex", + "snafu 0.6.10", +] + [[package]] name = "dyn-clone" version = "1.0.14" @@ -474,6 +526,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -607,6 +668,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -1308,6 +1379,51 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pest_meta" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -1363,8 +1479,8 @@ dependencies = [ [[package]] name = "product-config" -version = "0.5.0" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.5.0#439869d9e6a72fb6d912f6e494649a2f74f41d25" +version = "0.6.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.6.0#ad2c3ea6a291e415d978eb4271fb309e75861ef0" dependencies = [ "fancy-regex", "java-properties", @@ -1373,7 +1489,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "snafu 0.7.5", "xml-rs", ] @@ -1755,6 +1871,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.6" @@ -1788,6 +1915,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +[[package]] +name = "snafu" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab12d3c261b2308b0d80c26fffb58d17eba81a4be97890101f416b478c79ca7" +dependencies = [ + "doc-comment", + "snafu-derive 0.6.10", +] + [[package]] name = "snafu" version = "0.7.5" @@ -1795,7 +1932,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "snafu-derive", + "snafu-derive 0.7.5", +] + +[[package]] +name = "snafu-derive" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1508efa03c362e23817f96cde18abed596a25219a8b2c66e8db33c03543d315b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1845,7 +1993,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-operator", "strum", "tracing", @@ -1861,10 +2009,11 @@ dependencies = [ "fnv", "futures 0.3.28", "pin-project", + "product-config", "semver", "serde", "serde_json", - "snafu", + "snafu 0.7.5", "stackable-hive-crd", "stackable-operator", "strum", @@ -1874,13 +2023,14 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.55.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" +version = "0.56.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.0#1acaac4dacf302cc068b4294c8a1d4c2928977c0" dependencies = [ "chrono", "clap", "const_format", "derivative", + "dockerfile-parser", "either", "futures 0.3.28", "json-patch", @@ -1896,7 +2046,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-operator-derive", "strum", "thiserror", @@ -1908,8 +2058,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.55.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" +version = "0.56.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.0#1acaac4dacf302cc068b4294c8a1d4c2928977c0" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -2288,6 +2438,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/Cargo.toml b/Cargo.toml index d47bf017..a384074e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,5 @@ [workspace] -members = [ - "rust/crd", "rust/operator-binary" -] +members = ["rust/crd", "rust/operator-binary"] [workspace.package] version = "0.0.0-dev" @@ -12,7 +10,7 @@ repository = "https://github.com/stackabletech/hive-operator" [workspace.dependencies] anyhow = "1.0" -built = { version = "0.6", features = ["chrono", "git2"] } +built = { version = "0.6", features = ["chrono", "git2"] } clap = "4.3" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } @@ -24,7 +22,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.56.0" } +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.6.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 7ab2d40d..be3f961e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,6 +1,5 @@ -pub mod affinity; +use std::{collections::BTreeMap, str::FromStr}; -use affinity::get_affinity; use indoc::formatdoc; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; @@ -24,9 +23,12 @@ use stackable_operator::{ schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, }; -use std::{collections::BTreeMap, str::FromStr}; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; +use crate::affinity::get_affinity; + +pub mod affinity; + pub const APP_NAME: &str = "hive"; // directories pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 9c4e01c6..7b6963d9 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -21,6 +21,7 @@ serde.workspace = true serde_json.workspace = true snafu.workspace = true stackable-operator.workspace = true +product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7d61bf3d..7718c8ce 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,10 +1,18 @@ //! Ensures that `Pod`s are configured and running for each [`HiveCluster`] -use crate::command::{self, build_container_command_args, S3_SECRET_DIR}; -use crate::operations::pdb::add_pdbs; -use crate::product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}; -use crate::{discovery, OPERATOR_NAME}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, + hash::Hasher, + str::FromStr, + sync::Arc, +}; use fnv::FnvHasher; +use product_config::{ + types::PropertyNameKind, + writer::{to_hadoop_xml, to_java_properties_string, PropertiesWriterError}, + ProductConfigManager, +}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hive_crd::{ Container, DbType, HiveCluster, HiveClusterStatus, HiveRole, MetaStoreConfig, APP_NAME, @@ -14,9 +22,6 @@ use stackable_hive_crd::{ STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, STACKABLE_LOG_DIR_NAME, }; -use stackable_operator::k8s_openapi::DeepMerge; -use stackable_operator::memory::MemoryQuantity; -use stackable_operator::product_config::writer::to_java_properties_string; use stackable_operator::{ builder::{ resources::ResourceRequirementsBuilder, ConfigMapBuilder, ContainerBuilder, @@ -41,12 +46,12 @@ use stackable_operator::{ apimachinery::pkg::{ api::resource::Quantity, apis::meta::v1::LabelSelector, util::intstr::IntOrString, }, + DeepMerge, }, kube::{runtime::controller::Action, Resource, ResourceExt}, labels::{role_group_selector_labels, role_selector_labels, ObjectLabels}, logging::controller::ReconcilerError, - memory::BinaryMultiple, - product_config::{types::PropertyNameKind, ProductConfigManager}, + memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, product_logging::{ self, @@ -62,16 +67,17 @@ use stackable_operator::{ }, time::Duration, }; -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - hash::Hasher, - str::FromStr, - sync::Arc, -}; use strum::EnumDiscriminants; use tracing::warn; +use crate::{ + command::{self, build_container_command_args, S3_SECRET_DIR}, + discovery, + operations::pdb::add_pdbs, + product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}, + OPERATOR_NAME, +}; + /// Used as runAsUser in the pod security context. This is specified in the kafka image file pub const HIVE_UID: i64 = 1000; pub const HIVE_CONTROLLER_NAME: &str = "hivecluster"; @@ -93,112 +99,140 @@ pub struct Ctx { pub enum Error { #[snafu(display("object defines no namespace"))] ObjectHasNoNamespace, + #[snafu(display("object defines no metastore role"))] NoMetaStoreRole, + #[snafu(display("failed to calculate global service name"))] GlobalServiceNameNotFound, + #[snafu(display("failed to calculate service name for role {rolegroup}"))] RoleGroupServiceNameNotFound { rolegroup: RoleGroupRef, }, + #[snafu(display("failed to apply global Service"))] ApplyRoleService { source: stackable_operator::error::Error, }, + #[snafu(display("failed to apply Service for {rolegroup}"))] ApplyRoleGroupService { source: stackable_operator::error::Error, rolegroup: RoleGroupRef, }, + #[snafu(display("failed to build ConfigMap for {rolegroup}"))] BuildRoleGroupConfig { source: stackable_operator::error::Error, rolegroup: RoleGroupRef, }, + #[snafu(display("failed to apply ConfigMap for {rolegroup}"))] ApplyRoleGroupConfig { source: stackable_operator::error::Error, rolegroup: RoleGroupRef, }, + #[snafu(display("failed to apply StatefulSet for {rolegroup}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::error::Error, rolegroup: RoleGroupRef, }, + #[snafu(display("failed to generate product config"))] GenerateProductConfig { source: stackable_operator::product_config_utils::ConfigError, }, + #[snafu(display("invalid product config"))] InvalidProductConfig { source: stackable_operator::error::Error, }, + #[snafu(display("object is missing metadata to build owner reference"))] ObjectMissingMetadataForOwnerRef { source: stackable_operator::error::Error, }, + #[snafu(display("failed to build discovery ConfigMap"))] BuildDiscoveryConfig { source: discovery::Error }, + #[snafu(display("failed to apply discovery ConfigMap"))] ApplyDiscoveryConfig { source: stackable_operator::error::Error, }, + #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::error::Error, }, + #[snafu(display("failed to parse db type {db_type}"))] InvalidDbType { source: strum::ParseError, db_type: String, }, + #[snafu(display("failed to resolve S3 connection"))] ResolveS3Connection { source: stackable_operator::error::Error, }, + #[snafu(display( "Hive does not support skipping the verification of the tls enabled S3 server" ))] S3TlsNoVerificationNotSupported, + #[snafu(display("failed to resolve and merge resource config for role and role group"))] FailedToResolveResourceConfig { source: stackable_hive_crd::Error }, + #[snafu(display("invalid java heap config - missing default or value in crd?"))] InvalidJavaHeapConfig, + #[snafu(display("failed to convert java heap config to unit [{unit}]"))] FailedToConvertJavaHeap { source: stackable_operator::error::Error, unit: String, }, + #[snafu(display("failed to create hive container [{name}]"))] FailedToCreateHiveContainer { source: stackable_operator::error::Error, name: String, }, + #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::error::Error, }, + #[snafu(display("failed to delete orphaned resources"))] DeleteOrphanedResources { source: stackable_operator::error::Error, }, + #[snafu(display("failed to resolve the Vector aggregator address"))] ResolveVectorAggregatorAddress { source: crate::product_logging::Error, }, + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] InvalidLoggingConfig { source: crate::product_logging::Error, cm_name: String, }, + #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::error::Error, }, + #[snafu(display("failed to patch role binding"))] ApplyRoleBinding { source: stackable_operator::error::Error, }, + #[snafu(display("failed to build RBAC resources"))] BuildRbacResources { source: stackable_operator::error::Error, @@ -206,14 +240,16 @@ pub enum Error { #[snafu(display("internal operator failure"))] InternalOperatorError { source: stackable_hive_crd::Error }, + #[snafu(display( "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", rolegroup ))] JvmSecurityPoperties { - source: stackable_operator::product_config::writer::PropertiesWriterError, + source: PropertiesWriterError, rolegroup: String, }, + #[snafu(display("failed to create PodDisruptionBudget"))] FailedToCreatePdb { source: crate::operations::pdb::Error, @@ -563,8 +599,7 @@ fn build_metastore_rolegroup_config_map( data.insert(property_name.to_string(), Some(property_value.to_string())); } - hive_site_data = - stackable_operator::product_config::writer::to_hadoop_xml(data.iter()); + hive_site_data = to_hadoop_xml(data.iter()); } _ => {} }