diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e46705f..77ef92f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,11 @@ The broker `spec.celeryExecutors.broker` is now `spec.clusterConfig.celeryBroker`. - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#795]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#801]). +- Removed the product-config based configuration validation. + Config and environment overrides are now merged directly from the CRD, and dereferenced objects are added to the validated cluster. + The `--product-config` CLI flag is now a no-op ([#804]). +- The Flask config-file writer for `webserver_config.py` is now provided by `stackable-operator` + (`v2::flask_config_writer`) instead of a vendored copy ([#804]). ### Fixed @@ -43,6 +48,7 @@ [#795]: https://github.com/stackabletech/airflow-operator/pull/795 [#800]: https://github.com/stackabletech/airflow-operator/pull/800 [#801]: https://github.com/stackabletech/airflow-operator/pull/801 +[#804]: https://github.com/stackabletech/airflow-operator/pull/804 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.lock b/Cargo.lock index d6e8310c..0b263faf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1481,6 +1481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", "thiserror 1.0.69", @@ -1525,11 +1526,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1850,9 +1851,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1864,9 +1865,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1876,9 +1877,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1889,9 +1890,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1903,14 +1904,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1921,21 +1922,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2218,6 +2220,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2357,9 +2368,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2375,9 +2386,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2820,11 +2828,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2852,9 +2860,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2905,12 +2913,11 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", - "product-config", "rstest", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -2920,7 +2927,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -2932,7 +2939,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2943,8 +2950,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.111.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.111.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -2956,6 +2963,7 @@ dependencies = [ "futures 0.3.32", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -2968,7 +2976,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2980,12 +2988,14 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -2995,8 +3005,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -3005,15 +3015,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3024,7 +3034,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3037,21 +3047,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3069,7 +3079,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", @@ -3085,7 +3095,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3421,6 +3431,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3532,9 +3553,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3650,6 +3671,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3945,9 +3976,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" diff --git a/Cargo.nix b/Cargo.nix index 5ecf9a54..f3a57fca 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4714,6 +4714,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4729,6 +4734,10 @@ rec { } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4740,7 +4749,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4866,8 +4875,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "k8s_version"; @@ -4886,7 +4895,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -6067,9 +6076,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6106,24 +6115,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6166,18 +6175,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6213,16 +6219,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6283,10 +6289,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6311,16 +6316,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6333,27 +6341,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6397,30 +6405,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6428,9 +6435,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6456,6 +6463,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6480,10 +6494,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6500,15 +6522,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -7017,7 +7038,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7285,6 +7306,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7714,9 +7763,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7733,7 +7782,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7753,62 +7802,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7819,27 +7850,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7848,17 +7879,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -7867,33 +7898,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -7905,40 +7930,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9312,29 +9334,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9402,11 +9420,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9432,7 +9450,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9575,10 +9593,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "serde"; packageId = "serde"; @@ -9594,7 +9608,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9641,8 +9655,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_certs"; @@ -9701,7 +9715,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9740,12 +9754,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.111.0"; + version = "0.111.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_operator"; @@ -9796,6 +9810,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9803,6 +9821,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9852,7 +9871,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -9906,12 +9925,21 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -9924,8 +9952,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -9955,12 +9983,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_shared"; @@ -10005,7 +10033,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10036,12 +10064,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_telemetry"; @@ -10085,7 +10113,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10093,7 +10121,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10150,8 +10178,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_versioned"; @@ -10185,7 +10213,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10200,8 +10228,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -10268,8 +10296,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_webhook"; @@ -10342,7 +10370,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -11433,6 +11461,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.5"; + edition = "2021"; + sha256 = "16bk1cxi2m0xgaabf98nnj7dn9j16ymkh27jq4s3shjm4a85m1ra"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11889,9 +11944,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -12293,6 +12348,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -13951,9 +14066,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" diff --git a/Cargo.toml b/Cargo.toml index d04c1f1e..89d4bbc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,11 +10,10 @@ edition = "2021" repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = [ "crds", "webhook", -], tag = "stackable-operator-0.111.0" } +], tag = "stackable-operator-0.111.1" } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } @@ -34,5 +33,6 @@ tokio = { version = "1.40", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/crate-hashes.json b/crate-hashes.json index 71fbc1c3..014d9478 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/docs/modules/airflow/pages/reference/commandline-parameters.adoc b/docs/modules/airflow/pages/reference/commandline-parameters.adoc index 7a44b610..b41a07bf 100644 --- a/docs/modules/airflow/pages/reference/commandline-parameters.adoc +++ b/docs/modules/airflow/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/airflow-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-airflow-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/airflow/pages/reference/environment-variables.adoc b/docs/modules/airflow/pages/reference/environment-variables.adoc index 95824a7f..ea718826 100644 --- a/docs/modules/airflow/pages/reference/environment-variables.adoc +++ b/docs/modules/airflow/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/airflow-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/airflow-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-airflow-operator run ----- - -or via docker: - ----- -docker run \ - --name airflow-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/airflow-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces diff --git a/extra/crds.yaml b/extra/crds.yaml index 9ec7978a..691e3350 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -483,6 +483,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1001,6 +1002,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1998,6 +2000,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -2516,6 +2519,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -3069,6 +3073,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -3566,6 +3571,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -4084,6 +4090,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -4575,6 +4582,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -5093,6 +5101,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -5584,6 +5593,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -6107,6 +6117,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -6673,6 +6684,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -7191,6 +7203,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -8164,6 +8177,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -8682,6 +8696,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -9235,6 +9250,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -9732,6 +9748,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -10250,6 +10267,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -10741,6 +10759,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -11259,6 +11278,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -11750,6 +11770,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -12273,6 +12294,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 2734bb4e..92feeb34 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true anyhow.workspace = true diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index ea7dd9ad..ca1fcf65 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,16 +1,10 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`] use std::{ collections::{BTreeMap, BTreeSet, HashMap}, - io::Write, sync::Arc, }; use const_format::concatcp; -use product_config::{ - ProductConfigManager, - flask_app_config_writer::{self, FlaskAppConfigWriterError}, - types::PropertyNameKind, -}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -47,7 +41,7 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, PersistentVolumeClaim, PodTemplateSpec, Probe, ServiceAccount, + ConfigMap, EnvVar, PersistentVolumeClaim, PodTemplateSpec, Probe, ServiceAccount, TCPSocketAction, }, }, @@ -61,15 +55,7 @@ use stackable_operator::{ }, kvp::{Annotation, Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, - product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from, - env_vars_from_rolegroup_config, - }, - product_logging::{ - self, - framework::LoggingError, - spec::{ContainerLogConfig, Logging}, - }, + product_logging::{self, framework::LoggingError, spec::ContainerLogConfig}, role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ @@ -81,14 +67,14 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - config::{self, PYTHON_IMPORTS}, + controller::{build::config_map, validate::ValidatedAirflowCluster}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ - self, AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, - AirflowConfigOptions, AirflowExecutor, AirflowExecutorCommonConfiguration, AirflowRole, - CONFIG_PATH, Container, ExecutorConfig, HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, - LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, - STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + self, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowExecutor, + AirflowExecutorCommonConfiguration, AirflowRole, CONFIG_PATH, Container, ExecutorConfig, + HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, + METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, + TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -106,7 +92,6 @@ use crate::{ }, pdb::add_pdbs, }, - product_logging::extend_config_map_with_log_config, service::{ build_rolegroup_headless_service, build_rolegroup_metrics_service, stateful_set_service_name, @@ -120,7 +105,6 @@ pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -168,16 +152,9 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to build config file for {rolegroup}"))] - BuildRoleGroupConfigFile { - source: FlaskAppConfigWriterError, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("failed to build ConfigMap for {rolegroup}"))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + #[snafu(display("failed to build rolegroup ConfigMap"))] + BuildConfigMap { + source: crate::controller::build::config_map::Error, }, #[snafu(display("failed to resolve and merge config for role and role group"))] @@ -204,12 +181,6 @@ pub enum Error { #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::client::Error, @@ -261,14 +232,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("failed to construct config"))] - ConstructConfig { source: config::Error }, - - #[snafu(display( - "failed to write to String (Vec to be precise) containing Airflow config" - ))] - WriteToConfigFileString { source: std::io::Error }, - #[snafu(display("failed to configure logging"))] ConfigureLogging { source: LoggingError }, @@ -381,12 +344,10 @@ pub async fn reconcile_airflow( None }; - let validated = crate::controller::validate::validate_cluster( + let validated_cluster = crate::controller::validate::validate_cluster( airflow, - CONTAINER_IMAGE_BASE_NAME, &ctx.operator_environment.image_repository, - crate::built_info::PKG_VERSION, - &ctx.product_config, + dereferenced, ) .context(ValidateSnafu)?; @@ -457,15 +418,13 @@ pub async fn reconcile_airflow( // collection there will be a pod template created to be used for pod provisioning if let AirflowExecutor::KubernetesExecutors { common_configuration, - } = &validated.executor + } = &validated_cluster.executor { build_executor_template( airflow, common_configuration, &metadata_database_connection_details, - &validated.image, - &dereferenced.authentication_config, - &dereferenced.authorization_config, + &validated_cluster, &mut cluster_resources, client, &rbac_sa, @@ -473,10 +432,10 @@ pub async fn reconcile_airflow( .await?; } - for (airflow_role, role_group_configs) in &validated.role_groups { + for (airflow_role, role_group_configs) in &validated_cluster.role_groups { let role_name = airflow_role.to_string(); - if let Some(role_config) = validated.role_configs.get(airflow_role) { + if let Some(role_config) = validated_cluster.role_configs.get(airflow_role) { if let Some(pdb) = &role_config.pdb { add_pdbs(pdb, airflow, airflow_role, client, &mut cluster_resources) .await @@ -490,7 +449,7 @@ pub async fn reconcile_airflow( build_recommended_labels( airflow, AIRFLOW_CONTROLLER_NAME, - &validated.image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &role_name, "none", ), @@ -514,8 +473,17 @@ pub async fn reconcile_airflow( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, - &validated.image, - &env_vars_from_rolegroup_config(&validated_rg_config.product_config_properties), + &validated_cluster.image, + &validated_rg_config + .overrides + .env_overrides + .iter() + .map(|(k, v)| EnvVar { + name: k.clone(), + value: Some(v.clone()), + ..EnvVar::default() + }) + .collect::>(), &airflow.volume_mounts(), LOG_VOLUME_NAME, &validated_rg_config @@ -528,7 +496,7 @@ pub async fn reconcile_airflow( let role_group_service_recommended_labels = build_recommended_labels( airflow, AIRFLOW_CONTROLLER_NAME, - &validated.image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, ); @@ -570,16 +538,15 @@ pub async fn reconcile_airflow( rolegroup: rolegroup.clone(), })?; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = config_map::build_rolegroup_config_map( airflow, - &validated.image, + &validated_cluster, &rolegroup, - &validated_rg_config.product_config_properties, - &dereferenced.authentication_config, - &dereferenced.authorization_config, + &validated_rg_config.overrides.config_file_overrides, &validated_rg_config.merged_config.logging, &Container::Airflow, - )?; + ) + .context(BuildConfigMapSnafu)?; cluster_resources .add(client, rg_configmap) .await @@ -589,17 +556,17 @@ pub async fn reconcile_airflow( let rg_statefulset = build_server_rolegroup_statefulset( airflow, - &validated.image, + &validated_cluster.image, airflow_role, &rolegroup, - &validated_rg_config.product_config_properties, - &dereferenced.authentication_config, - &dereferenced.authorization_config, + &validated_rg_config.overrides.env_overrides, + &validated_cluster.authentication_config, + &validated_cluster.authorization_config, &metadata_database_connection_details, &celery_database_connection_details, &rbac_sa, &validated_rg_config.merged_config, - &validated.executor, + &validated_cluster.executor, &git_sync_resources, )?; @@ -639,9 +606,7 @@ async fn build_executor_template( airflow: &v1alpha2::AirflowCluster, common_config: &AirflowExecutorCommonConfiguration, metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - resolved_product_image: &ResolvedProductImage, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, + validated_cluster: &ValidatedAirflowCluster, cluster_resources: &mut ClusterResources<'_>, client: &stackable_operator::client::Client, rbac_sa: &stackable_operator::k8s_openapi::api::core::v1::ServiceAccount, @@ -655,16 +620,15 @@ async fn build_executor_template( role_group: "kubernetes".into(), }; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = config_map::build_rolegroup_config_map( airflow, - resolved_product_image, + validated_cluster, &rolegroup, - &HashMap::new(), - authentication_config, - authorization_config, + &BTreeMap::new(), &merged_executor_config.logging, &Container::Base, - )?; + ) + .context(BuildConfigMapSnafu)?; cluster_resources .add(client, rg_configmap) .await @@ -674,8 +638,16 @@ async fn build_executor_template( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, - resolved_product_image, - &env_vars_from(&common_config.env_overrides), + &validated_cluster.image, + &common_config + .env_overrides + .iter() + .map(|(k, v)| EnvVar { + name: k.clone(), + value: Some(v.clone()), + ..EnvVar::default() + }) + .collect::>(), &airflow.volume_mounts(), LOG_VOLUME_NAME, &merged_executor_config @@ -686,8 +658,8 @@ async fn build_executor_template( let worker_pod_template_config_map = build_executor_template_config_map( airflow, - resolved_product_image, - authentication_config, + &validated_cluster.image, + &validated_cluster.authentication_config, metadata_database_connection_details, &rbac_sa.name_unchecked(), &merged_executor_config, @@ -703,121 +675,6 @@ async fn build_executor_template( Ok(()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, - logging: &Logging, - container: &Container, -) -> Result { - let mut config: BTreeMap = BTreeMap::new(); - - // this will call default values from AirflowClientAuthenticationDetails - config::add_airflow_config( - &mut config, - authentication_config, - authorization_config, - &resolved_product_image.product_version, - ) - .context(ConstructConfigSnafu)?; - - tracing::debug!( - "Default config for {}: {:?}", - rolegroup.object_name(), - config - ); - - let mut file_config = rolegroup_config - .get(&PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.to_string())) - .cloned() - .unwrap_or_default(); - - tracing::debug!( - "Config overrides for {}: {:?}", - rolegroup.object_name(), - file_config - ); - - // now add any overrides, replacing any defaults - config.append(&mut file_config); - - tracing::debug!( - "Merged config for {}: {:?}", - rolegroup.object_name(), - config - ); - - let mut config_file = Vec::new(); - - // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) - if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; - } - - let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - - flask_app_config_writer::write::( - &mut config_file, - config.iter(), - PYTHON_IMPORTS, - ) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; - - if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; - } - - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(rolegroup.object_name()) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(ObjectMetaSnafu)? - .build(), - ) - .add_data( - AIRFLOW_CONFIG_FILENAME, - String::from_utf8(config_file).unwrap(), - ); - - extend_config_map_with_log_config( - rolegroup, - logging, - container, - &Container::Vector, - &mut cm_builder, - resolved_product_image, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - fn build_rolegroup_metadata( airflow: &v1alpha2::AirflowCluster, resolved_product_image: &&ResolvedProductImage, @@ -884,7 +741,7 @@ fn build_server_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, airflow_role: &AirflowRole, rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, + env_overrides: &HashMap, authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, @@ -974,7 +831,7 @@ fn build_server_rolegroup_statefulset( env_vars::build_airflow_statefulset_envs( airflow, airflow_role, - rolegroup_config, + env_overrides, executor, authentication_config, authorization_config, diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config/mod.rs similarity index 99% rename from rust/operator-binary/src/config.rs rename to rust/operator-binary/src/config/mod.rs index a3cd15ab..d4213c18 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,3 +1,5 @@ +pub mod webserver_config; + use std::collections::BTreeMap; use indoc::formatdoc; diff --git a/rust/operator-binary/src/config/webserver_config.rs b/rust/operator-binary/src/config/webserver_config.rs new file mode 100644 index 00000000..914e2a50 --- /dev/null +++ b/rust/operator-binary/src/config/webserver_config.rs @@ -0,0 +1,81 @@ +//! Builds the `webserver_config.py` Flask configuration file from the resolved +//! authentication/authorization config plus user-provided config overrides. + +use std::{collections::BTreeMap, io::Write}; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::v2::flask_config_writer; + +use super::{PYTHON_IMPORTS, add_airflow_config}; +use crate::crd::{ + AirflowConfigOptions, authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, +}; + +/// Marks arbitrary Python code to prepend verbatim to the generated file. +const CONFIG_OVERRIDE_FILE_HEADER_KEY: &str = "FILE_HEADER"; +/// Marks arbitrary Python code to append verbatim to the generated file. +const CONFIG_OVERRIDE_FILE_FOOTER_KEY: &str = "FILE_FOOTER"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to construct the webserver config"))] + ConstructConfig { source: super::Error }, + + #[snafu(display("failed to write the webserver config file"))] + WriteConfigFile { + source: flask_config_writer::FlaskAppConfigWriterError, + }, + + #[snafu(display("failed to write the header/footer to the webserver config file"))] + WriteHeaderFooter { source: std::io::Error }, +} + +/// Renders the `webserver_config.py` contents: operator defaults (derived from the +/// resolved authentication/authorization config) with the user's `config_overrides` +/// applied last, wrapped by the optional `FILE_HEADER`/`FILE_FOOTER` Python blocks. +pub fn build( + authentication_config: &AirflowClientAuthenticationDetailsResolved, + authorization_config: &AirflowAuthorizationResolved, + product_version: &str, + config_file_overrides: &BTreeMap, +) -> Result { + let mut config: BTreeMap = BTreeMap::new(); + + // this will call default values from AirflowClientAuthenticationDetails + add_airflow_config( + &mut config, + authentication_config, + authorization_config, + product_version, + ) + .context(ConstructConfigSnafu)?; + + let mut file_config = config_file_overrides.clone(); + + // now add any overrides, replacing any defaults + config.append(&mut file_config); + + let mut config_file = Vec::new(); + + // By removing the keys from `config`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).context(WriteHeaderFooterSnafu)?; + } + + let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + + flask_config_writer::write::( + &mut config_file, + config.iter(), + PYTHON_IMPORTS, + ) + .context(WriteConfigFileSnafu)?; + + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).context(WriteHeaderFooterSnafu)?; + } + + Ok(String::from_utf8(config_file).expect("the Flask config writer only emits valid UTF-8")) +} diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..b1bc88ab --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,109 @@ +//! Builds the rolegroup [`ConfigMap`]: the rendered `webserver_config.py` plus the +//! logging/vector configuration. + +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + k8s_openapi::api::core::v1::ConfigMap, + product_logging::spec::Logging, + role_utils::RoleGroupRef, +}; + +use crate::{ + airflow_controller::AIRFLOW_CONTROLLER_NAME, + config::webserver_config, + controller::validate::ValidatedAirflowCluster, + crd::{AIRFLOW_CONFIG_FILENAME, Container, build_recommended_labels, v1alpha2}, + product_logging::extend_config_map_with_log_config, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build webserver config for {rolegroup}"))] + BuildWebserverConfig { + source: webserver_config::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build object meta"))] + ObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, + + #[snafu(display("failed to build ConfigMap for {rolegroup}"))] + BuildConfigMap { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, +} + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] +pub fn build_rolegroup_config_map( + airflow: &v1alpha2::AirflowCluster, + validated_cluster: &ValidatedAirflowCluster, + rolegroup: &RoleGroupRef, + config_file_overrides: &BTreeMap, + logging: &Logging, + container: &Container, +) -> Result { + let config_file = webserver_config::build( + &validated_cluster.authentication_config, + &validated_cluster.authorization_config, + &validated_cluster.image.product_version, + config_file_overrides, + ) + .with_context(|_| BuildWebserverConfigSnafu { + rolegroup: rolegroup.clone(), + })?; + + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(airflow) + .name(rolegroup.object_name()) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + &validated_cluster.image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(ObjectMetaSnafu)? + .build(), + ) + .add_data(AIRFLOW_CONFIG_FILENAME, config_file); + + extend_config_map_with_log_config( + rolegroup, + logging, + container, + &Container::Vector, + &mut cm_builder, + &validated_cluster.image, + ) + .context(InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + })?; + + cm_builder.build().with_context(|_| BuildConfigMapSnafu { + rolegroup: rolegroup.clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs new file mode 100644 index 00000000..881a36f4 --- /dev/null +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -0,0 +1,3 @@ +//! Builders that assemble Kubernetes resources from the validated cluster. + +pub mod config_map; diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 1b261dfe..a1196d5d 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -1,2 +1,3 @@ +pub mod build; pub mod dereference; pub mod validate; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 6eb2a16c..c0ce093f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,18 +1,21 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; +use std::collections::BTreeMap; -use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self, ResolvedProductImage}, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; use strum::IntoEnumIterator; -use crate::crd::{AIRFLOW_CONFIG_FILENAME, AirflowConfig, AirflowExecutor, AirflowRole, v1alpha2}; +use super::dereference::DereferencedObjects; +use crate::{ + airflow_controller::CONTAINER_IMAGE_BASE_NAME, + crd::{ + AirflowConfig, AirflowExecutor, AirflowRole, MergedOverrides, + authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, v1alpha2, + }, +}; #[derive(Snafu, Debug)] pub enum Error { @@ -21,24 +24,8 @@ pub enum Error { source: product_image_selection::Error, }, - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("failed to generate product config"))] - GenerateProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("could not parse Airflow role [{role}]"))] - UnidentifiedAirflowRole { - source: strum::ParseError, - role: String, - }, } /// Per-role configuration extracted during validation. @@ -49,115 +36,101 @@ pub struct ValidatedRoleConfig { pub group_listener_name: Option, } -/// Per-rolegroup configuration: the merged CRD config plus the product-config properties. +/// Per-rolegroup configuration: the merged CRD config plus overrides. #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { pub merged_config: AirflowConfig, - pub product_config_properties: HashMap>, + pub overrides: MergedOverrides, } -/// The validated cluster: proves that product-config validation and config merging -/// succeeded for every role and role group before any resources are created. +/// The validated cluster: proves that config merging succeeded for every role and +/// role group before any resources are created. It also carries the dereferenced +/// external references, so every downstream build step reads them from here. #[derive(Clone, Debug)] pub struct ValidatedAirflowCluster { pub image: ResolvedProductImage, pub role_groups: BTreeMap>, pub role_configs: BTreeMap, pub executor: AirflowExecutor, + pub authentication_config: AirflowClientAuthenticationDetailsResolved, + pub authorization_config: AirflowAuthorizationResolved, } pub fn validate_cluster( airflow: &v1alpha2::AirflowCluster, - image_base_name: &str, image_repository: &str, - pkg_version: &str, - product_config_manager: &ProductConfigManager, + dereferenced: DereferencedObjects, ) -> Result { let resolved_product_image = airflow .spec .image - .resolve(image_base_name, image_repository, pkg_version) + .resolve( + CONTAINER_IMAGE_BASE_NAME, + image_repository, + crate::built_info::PKG_VERSION, + ) .context(ResolveProductImageSnafu)?; - let mut roles = HashMap::new(); + let mut role_groups = BTreeMap::new(); + let mut role_configs = BTreeMap::new(); // if the kubernetes executor is specified there will be no worker role as the pods // are provisioned by airflow as defined by the task (default: one pod per task) for role in AirflowRole::iter() { - if let Some(resolved_role) = airflow.get_role(&role) { - roles.insert( - role.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.into()), - ], - resolved_role.clone(), - ), - ); - } - } - - let role_config = transform_all_roles_to_config(airflow, &roles); - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config.context(GenerateProductConfigSnafu)?, - product_config_manager, - false, - false, - ) - .context(InvalidProductConfigSnafu)?; - - let mut role_groups = BTreeMap::new(); - let mut role_configs = BTreeMap::new(); - - for (role_name, rolegroup_configs) in validated_role_config.iter() { - let airflow_role = - AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { - role: role_name.to_string(), - })?; + let Some(resolved_role) = airflow.get_role(&role) else { + continue; + }; role_configs.insert( - airflow_role.clone(), + role.clone(), ValidatedRoleConfig { pdb: airflow - .role_config(&airflow_role) + .role_config(&role) .map(|rc| rc.pod_disruption_budget), - listener_class: airflow_role - .listener_class_name(airflow) - .map(|s| s.to_string()), - group_listener_name: airflow.group_listener_name(&airflow_role), + listener_class: role.listener_class_name(airflow).map(|s| s.to_string()), + group_listener_name: airflow.group_listener_name(&role), }, ); let mut group_configs = BTreeMap::new(); - for (rolegroup_name, rolegroup_config) in rolegroup_configs.iter() { + for rolegroup_name in resolved_role.role_groups.keys() { let rolegroup_ref = RoleGroupRef { cluster: stackable_operator::kube::runtime::reflector::ObjectRef::from_obj(airflow), - role: role_name.into(), + role: role.to_string(), role_group: rolegroup_name.into(), }; let merged_config = airflow - .merged_config(&airflow_role, &rolegroup_ref) + .merged_config(&role, &rolegroup_ref) + .context(FailedToResolveConfigSnafu)?; + + let overrides = airflow + .merged_overrides(&role, rolegroup_name) .context(FailedToResolveConfigSnafu)?; group_configs.insert( rolegroup_name.clone(), ValidatedRoleGroupConfig { merged_config, - product_config_properties: rolegroup_config.clone(), + overrides, }, ); } - role_groups.insert(airflow_role, group_configs); + role_groups.insert(role, group_configs); } + let DereferencedObjects { + authentication_config, + authorization_config, + } = dereferenced; + Ok(ValidatedAirflowCluster { image: resolved_product_image, role_groups, role_configs, executor: airflow.spec.executor.clone(), + authentication_config, + authorization_config, }) } diff --git a/rust/operator-binary/src/crd/authorization.rs b/rust/operator-binary/src/crd/authorization.rs index 2e15912c..0fc0726c 100644 --- a/rust/operator-binary/src/crd/authorization.rs +++ b/rust/operator-binary/src/crd/authorization.rs @@ -2,6 +2,7 @@ use stackable_operator::{client::Client, commons::opa::OpaApiVersion, shared::ti use crate::crd::{AirflowAuthorization, AirflowOpaConfig, v1alpha2}; +#[derive(Clone, Debug)] pub struct AirflowAuthorizationResolved { pub opa: Option, } @@ -24,6 +25,7 @@ impl AirflowAuthorizationResolved { } } +#[derive(Clone, Debug)] pub struct OpaConfigResolved { pub connection_string: String, pub cache_entry_time_to_live: Duration, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 83c54ef1..e7b228d6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,5 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; -use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -19,7 +18,6 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, - config_overrides::KeyValueConfigOverrides, crd::git_sync, deep_merger::ObjectOverrides, k8s_openapi::{ @@ -29,7 +27,6 @@ use stackable_operator::{ kube::{CustomResource, ResourceExt}, kvp::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, - product_config_utils::{self, Configuration}, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -42,6 +39,10 @@ use stackable_operator::{ shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, + v2::{ + config_overrides::KeyValueConfigOverrides, + flask_config_writer::{FlaskAppConfigOptions, PythonType}, + }, versioned::versioned, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; @@ -114,17 +115,10 @@ pub struct AirflowConfigOverrides { pub webserver_config_py: Option, } -impl stackable_operator::config_overrides::KeyValueOverridesProvider for AirflowConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - match file { - AIRFLOW_CONFIG_FILENAME => self - .webserver_config_py - .as_ref() - .map(|o| o.as_product_config_overrides()) - .unwrap_or_default(), - _ => BTreeMap::new(), - } - } +#[derive(Clone, Debug)] +pub struct MergedOverrides { + pub env_overrides: HashMap, + pub config_file_overrides: BTreeMap, } #[derive(Snafu, Debug)] @@ -450,6 +444,42 @@ impl v1alpha2::AirflowCluster { format!("{}-executor-pod-template", self.name_any()) } + pub fn merged_overrides( + &self, + role: &AirflowRole, + rolegroup_name: &str, + ) -> Result { + let role_config = role.role_config(self)?; + + let mut env_overrides = role_config.config.env_overrides.clone(); + // role-level webserver_config.py overrides; role-group overrides are extended on top below + // (role-group wins). Reads the v2 KeyValueConfigOverrides map directly, like hdfs/trino. + let mut file_overrides = role_config + .config + .config_overrides + .webserver_config_py + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + + if let Some(rg) = role_config.role_groups.get(rolegroup_name) { + env_overrides.extend(rg.config.env_overrides.clone()); + if let Some(rg_file) = rg.config.config_overrides.webserver_config_py.as_ref() { + file_overrides.extend(rg_file.overrides.clone()); + } + } + + let config_file_overrides = file_overrides + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + Ok(MergedOverrides { + env_overrides, + config_file_overrides, + }) + } + /// Retrieve and merge resource configs for role and role groups pub fn merged_config( &self, @@ -1003,35 +1033,6 @@ impl AirflowConfig { } } -impl Configuration for AirflowConfigFragment { - type Configurable = v1alpha2::AirflowCluster; - - fn compute_env( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_cli( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } -} - fn default_resources(role: &AirflowRole) -> ResourcesFragment { let (cpu, memory) = match role { AirflowRole::Worker => ( @@ -1114,13 +1115,15 @@ pub fn build_recommended_labels<'a, T>( #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use indoc::formatdoc; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, versioned::test_utils::RoundtripTestData, }; - use crate::{v1alpha1, v1alpha2}; + use crate::{crd::AirflowRole, v1alpha1, v1alpha2}; #[test] fn test_cluster_config() { @@ -1345,4 +1348,93 @@ mod tests { "# } } + + #[test] + fn merged_overrides_match_config_overrides_from_crd() { + let cluster_yaml = r#" + apiVersion: airflow.stackable.tech/v1alpha2 + kind: AirflowCluster + metadata: + name: airflow + spec: + image: + productVersion: 3.1.6 + clusterConfig: + loadExamples: false + exposeConfig: false + credentialsSecretName: airflow-admin-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + webservers: + config: {} + configOverrides: + webserver_config.py: + AUTH_TYPE: "AUTH_OID" + ROLE_ONLY_KEY: "role-value" + envOverrides: + ROLE_ENV_VAR: "role-env-value" + roleGroups: + default: + config: {} + configOverrides: + webserver_config.py: + AUTH_TYPE: "AUTH_DB" + GROUP_ONLY_KEY: "group-value" + envOverrides: + GROUP_ENV_VAR: "group-env-value" + schedulers: + config: {} + roleGroups: + default: + config: {} + kubernetesExecutors: + config: {} + "#; + + let deserializer = serde_yaml::Deserializer::from_str(cluster_yaml); + let cluster: v1alpha2::AirflowCluster = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + + // Webservers/default: role-group overrides merge with (and override) role-level ones + let overrides = cluster + .merged_overrides(&AirflowRole::Webserver, "default") + .expect("merged_overrides should succeed"); + + // configOverrides: group AUTH_TYPE overrides role AUTH_TYPE, both unique keys kept + assert_eq!( + overrides.config_file_overrides, + BTreeMap::from([ + ("AUTH_TYPE".into(), "AUTH_DB".into()), + ("ROLE_ONLY_KEY".into(), "role-value".into()), + ("GROUP_ONLY_KEY".into(), "group-value".into()), + ]) + ); + + // envOverrides: both role and group env vars present + assert_eq!(overrides.env_overrides.len(), 2); + assert_eq!( + overrides.env_overrides.get("ROLE_ENV_VAR").unwrap(), + "role-env-value" + ); + assert_eq!( + overrides.env_overrides.get("GROUP_ENV_VAR").unwrap(), + "group-env-value" + ); + + // Schedulers/default: no overrides configured → both maps empty + let overrides = cluster + .merged_overrides(&AirflowRole::Scheduler, "default") + .expect("merged_overrides should succeed"); + assert!( + overrides.config_file_overrides.is_empty(), + "scheduler should have no config file overrides" + ); + assert!( + overrides.env_overrides.is_empty(), + "scheduler should have no env overrides" + ); + } } diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index b4828dbe..595eb1ec 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -3,7 +3,6 @@ use std::{ path::PathBuf, }; -use product_config::types::PropertyNameKind; use snafu::Snafu; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, @@ -79,7 +78,7 @@ pub enum Error { pub fn build_airflow_statefulset_envs( airflow: &v1alpha2::AirflowCluster, airflow_role: &AirflowRole, - rolegroup_config: &HashMap>, + env_overrides: &HashMap, executor: &AirflowExecutor, auth_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, @@ -96,9 +95,6 @@ pub fn build_airflow_statefulset_envs( env.extend(static_envs(git_sync_resources)); - // environment variables - let env_vars = rolegroup_config.get(&PropertyNameKind::Env); - add_version_specific_env_vars(airflow, airflow_role, resolved_product_image, &mut env); // N.B. this has been deprecated and replaced with AIRFLOW__API__SECRET_KEY since 3.0.2. Can be removed when 3.0.1 is no longer supported. @@ -266,17 +262,15 @@ pub fn build_airflow_statefulset_envs( } // apply overrides last of all with a fixed ordering - if let Some(env_vars) = env_vars { - for (k, v) in env_vars.iter().collect::>() { - env.insert( - k.into(), - EnvVar { - name: k.to_string(), - value: Some(v.to_string()), - ..Default::default() - }, - ); - } + for (k, v) in env_overrides.iter().collect::>() { + env.insert( + k.into(), + EnvVar { + name: k.to_string(), + value: Some(v.to_string()), + ..Default::default() + }, + ); } // Needed for the `containerdebug` process to log it's tracing information to. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index dee5603b..469a7240 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -71,11 +71,11 @@ async fn main() -> anyhow::Result<()> { .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; } Command::Run(RunArguments { - product_config, watch_namespace, operator_environment, maintenance, common, + .. }) => { // NOTE (@NickLarsenNZ): Before stackable-telemetry was used: // - The console log level was set by `AIRFLOW_OPERATOR_LOG`, and is now `CONSOLE_LOG` (when using Tracing::pre_configured). @@ -103,11 +103,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map(anyhow::Ok); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/airflow-operator/config-spec/properties.yaml", - ])?; - let client = stackable_operator::client::initialize_operator( Some(OPERATOR_NAME.to_string()), &common.cluster_info, @@ -183,7 +178,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(airflow_controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background