diff --git a/Cargo.lock b/Cargo.lock index 54be2875..a0b1e7da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,6 +195,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "built" version = "0.6.1" @@ -347,6 +356,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -366,6 +384,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.14.4" @@ -447,12 +475,36 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dockerfile-parser" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75799314f5fa405629a365a1f97d80f81edd17f22a0fc9c8ddb3ad191ad8dc08" +dependencies = [ + "enquote", + "lazy_static", + "pest", + "pest_derive", + "regex", + "snafu 0.6.10", +] + [[package]] name = "dyn-clone" version = "1.0.14" @@ -474,6 +526,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -607,6 +668,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -1302,6 +1373,51 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pest_meta" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -1357,8 +1473,8 @@ dependencies = [ [[package]] name = "product-config" -version = "0.5.0" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.5.0#439869d9e6a72fb6d912f6e494649a2f74f41d25" +version = "0.6.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.6.0#ad2c3ea6a291e415d978eb4271fb309e75861ef0" dependencies = [ "fancy-regex", "java-properties", @@ -1367,7 +1483,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "snafu 0.7.5", "xml-rs", ] @@ -1749,6 +1865,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.6" @@ -1782,6 +1909,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +[[package]] +name = "snafu" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab12d3c261b2308b0d80c26fffb58d17eba81a4be97890101f416b478c79ca7" +dependencies = [ + "doc-comment", + "snafu-derive 0.6.10", +] + [[package]] name = "snafu" version = "0.7.5" @@ -1789,7 +1926,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "snafu-derive", + "snafu-derive 0.7.5", +] + +[[package]] +name = "snafu-derive" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1508efa03c362e23817f96cde18abed596a25219a8b2c66e8db33c03543d315b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1834,11 +1982,12 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" name = "stackable-airflow-crd" version = "0.0.0-dev" dependencies = [ + "product-config", "rstest", "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-operator", "strum", "tracing", @@ -1853,9 +2002,10 @@ dependencies = [ "clap", "fnv", "futures 0.3.28", + "product-config", "serde", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-airflow-crd", "stackable-operator", "strum", @@ -1865,13 +2015,14 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.55.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" +version = "0.56.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.0#1acaac4dacf302cc068b4294c8a1d4c2928977c0" dependencies = [ "chrono", "clap", "const_format", "derivative", + "dockerfile-parser", "either", "futures 0.3.28", "json-patch", @@ -1887,7 +2038,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu", + "snafu 0.7.5", "stackable-operator-derive", "strum", "thiserror", @@ -1899,8 +2050,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.55.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" +version = "0.56.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.0#1acaac4dacf302cc068b4294c8a1d4c2928977c0" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -2279,6 +2430,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/Cargo.toml b/Cargo.toml index 035b0985..35a4c35a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,5 @@ [workspace] -members = [ - "rust/crd", "rust/operator-binary" -] +members = ["rust/crd", "rust/operator-binary"] [workspace.package] version = "0.0.0-dev" @@ -12,7 +10,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] anyhow = "1.0" -built = { version = "0.6", features = ["chrono", "git2"] } +built = { version = "0.6", features = ["chrono", "git2"] } clap = "4.3" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } @@ -22,7 +20,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.56.0" } +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.6.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index f4c2c6c8..4548c22e 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -13,6 +13,7 @@ serde.workspace = true serde_json.workspace = true snafu.workspace = true stackable-operator.workspace = true +product-config.workspace = true strum.workspace = true tracing.workspace = true diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 005fc7f0..452f94f9 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,11 +1,7 @@ -pub mod affinity; -pub mod authentication; -mod git_sync; - -use crate::affinity::get_affinity; -use crate::authentication::AirflowAuthentication; +use std::collections::BTreeMap; use git_sync::GitSync; +use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -26,16 +22,20 @@ use stackable_operator::{ kube::{CustomResource, ResourceExt}, labels::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, - product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}, product_config_utils::{ConfigError, Configuration}, product_logging::{self, spec::Logging}, role_utils::{CommonConfiguration, GenericRoleConfig, Role, RoleGroup, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, }; -use std::collections::BTreeMap; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; +use crate::{affinity::get_affinity, authentication::AirflowAuthentication}; + +pub mod affinity; +pub mod authentication; +mod git_sync; + pub const AIRFLOW_UID: i64 = 1000; pub const APP_NAME: &str = "airflow"; pub const OPERATOR_NAME: &str = "airflow.stackable.tech"; diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index fefaf2e4..fdd08d96 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -19,6 +19,7 @@ serde.workspace = true serde_yaml.workspace = true snafu.workspace = true stackable-operator.workspace = true +product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index fc8842e3..f7f22e7e 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,39 +1,28 @@ //! Ensures that `Pod`s are configured and running for each [`AirflowCluster`] -use stackable_operator::builder::resources::ResourceRequirementsBuilder; -use stackable_operator::config::fragment::ValidationError; -use stackable_operator::k8s_openapi::DeepMerge; -use stackable_operator::product_logging::framework::{ - capture_shell_output, shutdown_vector_command, +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, + sync::Arc, }; -use stackable_operator::product_logging::spec::{ContainerLogConfig, ContainerLogConfigChoice}; -use stackable_operator::role_utils::GenericRoleConfig; -use crate::config::{self, PYTHON_IMPORTS}; -use crate::controller_commons::{ - self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME, -}; -use crate::operations::pdb::add_pdbs; -use crate::product_logging::{ - extend_config_map_with_log_config, resolve_vector_aggregator_address, +use product_config::{ + flask_app_config_writer::{self, FlaskAppConfigWriterError}, + types::PropertyNameKind, + ProductConfigManager, }; -use crate::util::env_var_from_secret; - use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_airflow_crd::authentication::AirflowAuthenticationConfigResolved; -use stackable_airflow_crd::{ - build_recommended_labels, AirflowCluster, AirflowConfig, AirflowConfigFragment, - AirflowConfigOptions, AirflowRole, Container, AIRFLOW_CONFIG_FILENAME, APP_NAME, CONFIG_PATH, - LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, -}; use stackable_airflow_crd::{ - AirflowClusterStatus, AirflowExecutor, ExecutorConfig, AIRFLOW_UID, GIT_CONTENT, GIT_LINK, - GIT_ROOT, GIT_SYNC_DIR, GIT_SYNC_NAME, TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, + authentication::AirflowAuthenticationConfigResolved, build_recommended_labels, AirflowCluster, + AirflowClusterStatus, AirflowConfig, AirflowConfigFragment, AirflowConfigOptions, + AirflowExecutor, AirflowRole, Container, ExecutorConfig, AIRFLOW_CONFIG_FILENAME, AIRFLOW_UID, + APP_NAME, CONFIG_PATH, GIT_CONTENT, GIT_LINK, GIT_ROOT, GIT_SYNC_DIR, GIT_SYNC_NAME, + LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, }; use stackable_operator::{ builder::{ - ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, - PodSecurityContextBuilder, VolumeBuilder, + resources::ResourceRequirementsBuilder, ConfigMapBuilder, ContainerBuilder, + ObjectMetaBuilder, PodBuilder, PodSecurityContextBuilder, VolumeBuilder, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ @@ -41,6 +30,7 @@ use stackable_operator::{ product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, }, + config::fragment::ValidationError, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, @@ -50,6 +40,7 @@ use stackable_operator::{ }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + DeepMerge, }, kube::{ runtime::{controller::Action, reflector::ObjectRef}, @@ -57,26 +48,29 @@ use stackable_operator::{ }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, - product_config::{ - flask_app_config_writer, flask_app_config_writer::FlaskAppConfigWriterError, - types::PropertyNameKind, ProductConfigManager, - }, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, - product_logging::{self, spec::Logging}, - role_utils::RoleGroupRef, + product_logging::{ + self, + framework::{capture_shell_output, create_vector_shutdown_file_command}, + spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, + }, + role_utils::{GenericRoleConfig, RoleGroupRef}, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, time::Duration, }; -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, - sync::Arc, -}; use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; +use crate::{ + config::{self, PYTHON_IMPORTS}, + controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + operations::pdb::add_pdbs, + product_logging::{extend_config_map_with_log_config, resolve_vector_aggregator_address}, + util::env_var_from_secret, +}; + pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; pub const DOCKER_IMAGE_BASE_NAME: &str = "airflow"; @@ -984,7 +978,7 @@ fn build_executor_template_config_map( [ // Wait for Vector to gather the logs. "sleep 10", - &shutdown_vector_command(STACKABLE_LOG_DIR), + &create_vector_shutdown_file_command(STACKABLE_LOG_DIR), ] .join("; "), );