From 5cb377b2763cb96553134871af1eb0f66d40c613 Mon Sep 17 00:00:00 2001 From: Xenia Date: Mon, 26 Aug 2024 18:58:18 +0200 Subject: [PATCH 1/4] chore: add overrides integrationtests (#490) * chore: add hint for pod template * recapture pod_template to immutable * add overrides integrationtest * fix overrides integrationtest * bump stackable-operator version * add todo key * remove recapturing for now * remove recapturing for now * test env overrides for celery executors * reorganize steps * test env overrides for k8s executors * fix yaml lint --------- Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> --- Cargo.lock | 145 +++++++------- Cargo.nix | 185 ++++++++++-------- Cargo.toml | 2 +- crate-hashes.json | 4 +- .../kuttl/overrides/00-assert.yaml.j2 | 20 ++ .../kuttl/overrides/00-install-dbs.yaml.j2 | 21 ++ .../templates/kuttl/overrides/10-assert.yaml | 8 + .../overrides/10-install-airflow.yaml.j2 | 99 ++++++++++ .../templates/kuttl/overrides/11-assert.yaml | 19 ++ .../templates/kuttl/overrides/20-assert.yaml | 8 + .../overrides/20-install-airflow2.yaml.j2 | 66 +++++++ .../templates/kuttl/overrides/21-assert.yaml | 8 + tests/test-definition.yaml | 4 + 13 files changed, 435 insertions(+), 154 deletions(-) create mode 100644 tests/templates/kuttl/overrides/00-assert.yaml.j2 create mode 100644 tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 create mode 100644 tests/templates/kuttl/overrides/10-assert.yaml create mode 100644 tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 create mode 100644 tests/templates/kuttl/overrides/11-assert.yaml create mode 100644 tests/templates/kuttl/overrides/20-assert.yaml create mode 100644 tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 create mode 100644 tests/templates/kuttl/overrides/21-assert.yaml diff --git a/Cargo.lock b/Cargo.lock index 112b680b..960a6ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -157,7 +157,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -270,12 +270,13 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -299,9 +300,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.15" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -328,7 +329,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -390,9 +391,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -443,7 +444,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -454,7 +455,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -465,7 +466,7 @@ checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -672,7 +673,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -991,9 +992,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown", @@ -1048,9 +1049,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -1185,7 +1186,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1224,9 +1225,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libgit2-sys" @@ -1242,9 +1243,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" dependencies = [ "cc", "libc", @@ -1524,7 +1525,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1555,7 +1556,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1764,7 +1765,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.74", + "syn 2.0.75", "unicode-ident", ] @@ -1800,9 +1801,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -1881,7 +1882,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1931,9 +1932,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.206" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -1950,13 +1951,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.206" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1967,14 +1968,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] name = "serde_json" -version = "1.0.124" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "itoa", "memchr", @@ -2026,6 +2027,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -2089,7 +2096,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2145,8 +2152,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.73.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +version = "0.74.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "chrono", "clap", @@ -2159,7 +2166,6 @@ dependencies = [ "json-patch", "k8s-openapi", "kube", - "lazy_static", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2183,12 +2189,12 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2216,7 +2222,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2238,9 +2244,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.74" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -2264,7 +2270,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2347,9 +2353,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", @@ -2371,7 +2377,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2465,15 +2471,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -2507,7 +2513,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2621,9 +2627,9 @@ dependencies = [ [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" [[package]] name = "unsafe-libyaml" @@ -2690,34 +2696,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2725,22 +2732,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-time" @@ -2889,7 +2896,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] diff --git a/Cargo.nix b/Cargo.nix index 43d079b2..b83c84f2 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -446,7 +446,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "visit-mut" ]; } ]; @@ -473,7 +473,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "full" "visit-mut" "parsing" "printing" "proc-macro" ]; } @@ -776,9 +776,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.1.10"; + version = "1.1.13"; edition = "2018"; - sha256 = "14jjp993s17yfvl5linq3gs4jzb7f42nkq09r5kyfjskmjzsms79"; + sha256 = "0j6czmvfj9qhqmazznvg2wkzlyh7p65pwdqhnl1rpvpc8xwjznvj"; authors = [ "Alex Crichton " ]; @@ -796,6 +796,10 @@ rec { usesDefaultFeatures = false; target = { target, features }: (target."unix" or false); } + { + name = "shlex"; + packageId = "shlex"; + } ]; features = { "parallel" = [ "dep:libc" "dep:jobserver" ]; @@ -880,10 +884,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.5.15"; + version = "4.5.16"; edition = "2021"; crateBin = []; - sha256 = "1k656mr99srcla2bx2h7wjwlb9mywavb5k1szpd5b9pxaj287n0i"; + sha256 = "068hjwbrndn4iz4fsc6d52q4ymg1kfsymjnqbxzdil23zbzijrzd"; dependencies = [ { name = "clap_builder"; @@ -979,7 +983,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -1139,9 +1143,9 @@ rec { }; "cpufeatures" = rec { crateName = "cpufeatures"; - version = "0.2.12"; + version = "0.2.13"; edition = "2018"; - sha256 = "012m7rrak4girqlii3jnqwrr73gv1i980q4wra5yyyhvzwk5xzjk"; + sha256 = "1b89kljf7phyh63vxwsvf9lbgwkv0dsj7pcjmqgysnwsvkk55s2i"; authors = [ "RustCrypto Developers" ]; @@ -1283,7 +1287,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "extra-traits" ]; } ]; @@ -1313,7 +1317,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; @@ -1339,7 +1343,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "visit-mut" ]; } ]; @@ -1884,7 +1888,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -2945,9 +2949,9 @@ rec { }; "indexmap" = rec { crateName = "indexmap"; - version = "2.3.0"; + version = "2.4.0"; edition = "2021"; - sha256 = "182yvyck0zk3swa9bs8l9a8lmis3q7zy338wj6rx2bd81giw4gyy"; + sha256 = "0p2hwvmir50qcl5q6lib8fjq5dzv4f0gqy8czcyfva3yzhzdbslk"; dependencies = [ { name = "equivalent"; @@ -3076,9 +3080,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.69"; - edition = "2018"; - sha256 = "0v99rz97asnzapb0jsc3jjhvxpfxr7h7qd97yqyrf9i7viimbh99"; + version = "0.3.70"; + edition = "2021"; + sha256 = "0yp3rz7vrn9mmqdpkds426r1p9vs6i8mkxx8ryqdfadr0s2q0s0q"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -3678,7 +3682,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "extra-traits" ]; } ]; @@ -3820,9 +3824,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.155"; + version = "0.2.158"; edition = "2015"; - sha256 = "0z44c53z54znna8n322k5iwg80arxxpdzjj5260pxxzc9a58icwp"; + sha256 = "0fb4qldw1jrxljrwz6bsjn8lv4rqizlqmab41q3j98q332xw9bfq"; authors = [ "The Rust Project Developers" ]; @@ -3880,10 +3884,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.18"; + version = "1.1.19"; edition = "2018"; links = "z"; - sha256 = "0bpqmfzvijbrqs29vphnafjz834lpz6pabbsnf85rqppb9pa4pf1"; + sha256 = "0is6lzp9xrgryn57d6ajvadkrs40vzvk37yjkyy6wjd7k5vkmigx"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -4773,7 +4777,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; features = { @@ -4846,7 +4850,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "visit-mut" ]; } ]; @@ -5483,7 +5487,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ]; } { @@ -5605,9 +5609,9 @@ rec { }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; - version = "0.7.1"; + version = "0.7.2"; edition = "2021"; - sha256 = "1fjzn7h3p6f3wr014p0i58131513b5li3146v6yrdd2i0r16v3d8"; + sha256 = "1ynz8a108bd3k7zix26z79n8cnqfsrfflsf01x7406h9r7zjs604"; libName = "rustls_native_certs"; dependencies = [ { @@ -5848,7 +5852,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "extra-traits" ]; } ]; @@ -5988,9 +5992,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.206"; + version = "1.0.208"; edition = "2018"; - sha256 = "1102kmaf7cdln5f44l1ygqz457ly9pii3kcb28555p9387clqgjv"; + sha256 = "1cng2zkvv6hh137jrrqdmhi2hllfnqwlqjgviqjalkv8rg98bw6g"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6043,9 +6047,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.206"; + version = "1.0.208"; edition = "2015"; - sha256 = "15rgfrcs56hqhhfnhq4413sazcfzkkvcr66g121fm0r3il9vdgzs"; + sha256 = "1krblvy84j6d9zj12ms9l2wfbwf2w8jazkx0bf7fs4vnzy0qw014"; procMacro = true; authors = [ "Erick Tryzelaar " @@ -6066,7 +6070,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -6098,7 +6102,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -6107,9 +6111,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.124"; + version = "1.0.125"; edition = "2021"; - sha256 = "13g3zka165qqnvbw0c6k708wdfc4738sr4f8ijjxpcsnga265bb6"; + sha256 = "1v9idlv5nq5f000qsv6v02h2zbsfk2m6d05af2zgbk3kl0sygj43"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6279,6 +6283,24 @@ rec { "loom" = [ "dep:loom" ]; }; }; + "shlex" = rec { + crateName = "shlex"; + version = "1.3.0"; + edition = "2015"; + sha256 = "0r1y6bv26c1scpxvhg2cabimrmwgbp4p3wy6syj9n0c4s3q2znhg"; + authors = [ + "comex " + "Fenhl " + "Adrian Taylor " + "Alex Touchet " + "Daniel Parks " + "Garrett Berg " + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "signal-hook-registry" = rec { crateName = "signal-hook-registry"; version = "1.4.2"; @@ -6451,7 +6473,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -6653,13 +6675,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.73.0"; + version = "0.74.0"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "4d98a29b08a7d959e5e287f774cf064c02ffbd62"; - sha256 = "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m"; + rev = "c77a5423b66bc1667b63af7d8bec00de88a5303f"; + sha256 = "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya"; }; libName = "stackable_operator"; authors = [ @@ -6716,10 +6738,6 @@ rec { usesDefaultFeatures = false; features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; } - { - name = "lazy_static"; - packageId = "lazy_static"; - } { name = "opentelemetry-jaeger"; packageId = "opentelemetry-jaeger"; @@ -6812,8 +6830,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "4d98a29b08a7d959e5e287f774cf064c02ffbd62"; - sha256 = "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m"; + rev = "c77a5423b66bc1667b63af7d8bec00de88a5303f"; + sha256 = "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -6835,7 +6853,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; @@ -6908,7 +6926,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "parsing" "extra-traits" ]; } ]; @@ -6961,11 +6979,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit" ]; }; - "syn 2.0.74" = rec { + "syn 2.0.75" = rec { crateName = "syn"; - version = "2.0.74"; + version = "2.0.75"; edition = "2021"; - sha256 = "1mxk5h27mzl0r2j434rqrm279k302jdl1lr16kc0nval7lgb9khz"; + sha256 = "1nf72xc8pabwny8d0d4kjk5vmadg3fa240dywznka6gw6hq0dbzn"; authors = [ "David Tolnay " ]; @@ -7031,7 +7049,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; @@ -7270,9 +7288,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.39.2"; + version = "1.39.3"; edition = "2021"; - sha256 = "1cb7yhba7nnf00cylcywk7rai5kkdb8b4jzwrc26zgbqqwdzp96s"; + sha256 = "1xgzhj7bxqqpjaabjkgsx8hi0f600bzj4iyp9f0a9gr3k6dwkawv"; authors = [ "Tokio Contributors " ]; @@ -7393,7 +7411,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -7801,9 +7819,9 @@ rec { }; "tower-layer" = rec { crateName = "tower-layer"; - version = "0.3.2"; + version = "0.3.3"; edition = "2018"; - sha256 = "1l7i17k9vlssrdg4s3b0ia5jjkmmxsvv8s9y9ih0jfi8ssz8s362"; + sha256 = "03kq92fdzxin51w8iqix06dcfgydyvx7yr6izjq0p626v9n2l70j"; libName = "tower_layer"; authors = [ "Tower Maintainers " @@ -7812,9 +7830,9 @@ rec { }; "tower-service" = rec { crateName = "tower-service"; - version = "0.3.2"; + version = "0.3.3"; edition = "2018"; - sha256 = "0lmfzmmvid2yp2l36mbavhmqgsvzqf7r2wiwz73ml4xmwaf1rg5n"; + sha256 = "1hzfkvkci33ra94xjx64vv3pp0sq346w06fpkcdwjcid7zhvdycd"; libName = "tower_service"; authors = [ "Tower Maintainers " @@ -7927,7 +7945,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -8344,9 +8362,9 @@ rec { }; "unicode-xid" = rec { crateName = "unicode-xid"; - version = "0.2.4"; + version = "0.2.5"; edition = "2015"; - sha256 = "131dfzf7d8fsr1ivch34x42c2d1ik5ig3g78brxncnn0r1sdyqpr"; + sha256 = "02n96yhcjwx7vphbwf9p7xfqbwvqgzdw2qz4h0x3wd5wgxj315r2"; libName = "unicode_xid"; authors = [ "erick.tryzelaar " @@ -8500,9 +8518,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1a4mcw13nsk3fr8fxjzf9kk1wj88xkfsmnm0pjraw01ryqfm7qjb"; + version = "0.2.93"; + edition = "2021"; + sha256 = "1dfr7pka5kwvky2fx82m9d060p842hc5fyyw8igryikcdb0xybm8"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -8512,6 +8530,10 @@ rec { name = "cfg-if"; packageId = "cfg-if"; } + { + name = "once_cell"; + packageId = "once_cell"; + } { name = "wasm-bindgen-macro"; packageId = "wasm-bindgen-macro"; @@ -8520,7 +8542,6 @@ rec { features = { "default" = [ "spans" "std" ]; "enable-interning" = [ "std" ]; - "gg-alloc" = [ "wasm-bindgen-test/gg-alloc" ]; "serde" = [ "dep:serde" ]; "serde-serialize" = [ "serde" "serde_json" "std" ]; "serde_json" = [ "dep:serde_json" ]; @@ -8532,9 +8553,9 @@ rec { }; "wasm-bindgen-backend" = rec { crateName = "wasm-bindgen-backend"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1nj7wxbi49f0rw9d44rjzms26xlw6r76b2mrggx8jfbdjrxphkb1"; + version = "0.2.93"; + edition = "2021"; + sha256 = "0yypblaf94rdgqs5xw97499xfwgs1096yx026d6h88v563d9dqwx"; libName = "wasm_bindgen_backend"; authors = [ "The wasm-bindgen Developers" @@ -8562,7 +8583,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } { @@ -8577,9 +8598,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.92"; - edition = "2018"; - sha256 = "09npa1srjjabd6nfph5yc03jb26sycjlxhy0c2a1pdrpx4yq5y51"; + version = "0.2.93"; + edition = "2021"; + sha256 = "1kycd1xfx4d9xzqknvzbiqhwb5fzvjqrrn88x692q1vblj8lqp2q"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -8603,9 +8624,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1dqv2xs8zcyw4kjgzj84bknp2h76phmsb3n7j6hn396h4ssifkz9"; + version = "0.2.93"; + edition = "2021"; + sha256 = "0dp8w6jmw44srym6l752nkr3hkplyw38a2fxz5f3j1ch9p3l1hxg"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -8621,7 +8642,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "visit" "full" ]; } { @@ -8641,10 +8662,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.92"; - edition = "2018"; + version = "0.2.93"; + edition = "2021"; links = "wasm_bindgen"; - sha256 = "15kyavsrna2cvy30kg03va257fraf9x00ny554vxngvpyaa0q6dg"; + sha256 = "1104bny0hv40jfap3hp8jhs0q4ya244qcrvql39i38xlghq0lan6"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -9218,7 +9239,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; diff --git a/Cargo.toml b/Cargo.toml index 190cef64..9a5a8100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.39", features = ["full"] } tracing = "0.1" diff --git a/crate-hashes.json b/crate-hashes.json index ddd91d81..8725a492 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,5 +1,5 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#stackable-operator-derive@0.3.1": "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#stackable-operator@0.73.0": "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#stackable-operator-derive@0.3.1": "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#stackable-operator@0.74.0": "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/tests/templates/kuttl/overrides/00-assert.yaml.j2 b/tests/templates/kuttl/overrides/00-assert.yaml.j2 new file mode 100644 index 00000000..4b8ea1bb --- /dev/null +++ b/tests/templates/kuttl/overrides/00-assert.yaml.j2 @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-master +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-replicas +status: + availableReplicas: 1 \ No newline at end of file diff --git a/tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 b/tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 new file mode 100644 index 00000000..f98f9be8 --- /dev/null +++ b/tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-postgresql + --namespace $NAMESPACE + --version 12.1.5 + --set auth.username=airflow + --set auth.password=airflow + --set auth.database=airflow + --repo https://charts.bitnami.com/bitnami postgresql + timeout: 240 + - script: >- + helm install airflow-redis + --namespace $NAMESPACE + --version 17.3.7 + --set auth.password=redis + --set replica.replicaCount=1 + --repo https://charts.bitnami.com/bitnami redis + timeout: 240 \ No newline at end of file diff --git a/tests/templates/kuttl/overrides/10-assert.yaml b/tests/templates/kuttl/overrides/10-assert.yaml new file mode 100644 index 00000000..fb129c43 --- /dev/null +++ b/tests/templates/kuttl/overrides/10-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-celery --timeout 301s diff --git a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 new file mode 100644 index 00000000..2e7da7ff --- /dev/null +++ b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 @@ -0,0 +1,99 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-override +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow-celery +spec: + image: + productVersion: "{{ test_scenario['values']['airflow-latest'] }}" + clusterConfig: + loadExamples: true + exposeConfig: false + listenerClass: external-unstable + credentialsSecret: airflow-credentials + webservers: + envOverrides: + COMMON_VAR: role-value # overridden by role group below + ROLE_VAR: role-value # only defined here at role level + credentialsSecret: test-override + roleGroups: + default: + replicas: 1 + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + celeryExecutors: + envOverrides: + COMMON_VAR: role-value # overridden by role group below + ROLE_VAR: role-value # only defined here at role level + roleGroups: + default: + replicas: 2 + config: + resources: + cpu: + min: 410m + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 810m + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + config: + resources: + cpu: + min: 400m + max: 800m + memory: + limit: 2Gi + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 750m + requests: + cpu: 350m + schedulers: + roleGroups: + default: + replicas: 1 + envOverrides: + credentialsSecret: test-override diff --git a/tests/templates/kuttl/overrides/11-assert.yaml b/tests/templates/kuttl/overrides/11-assert.yaml new file mode 100644 index 00000000..33645a7a --- /dev/null +++ b/tests/templates/kuttl/overrides/11-assert.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # + # Test envOverrides + # + - script: | + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "credentialsSecret" and .value == "test-override")' + + + - script: | + kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' diff --git a/tests/templates/kuttl/overrides/20-assert.yaml b/tests/templates/kuttl/overrides/20-assert.yaml new file mode 100644 index 00000000..22025a66 --- /dev/null +++ b/tests/templates/kuttl/overrides/20-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-kubernetes --timeout 301s diff --git a/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 new file mode 100644 index 00000000..bbaf47b4 --- /dev/null +++ b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 @@ -0,0 +1,66 @@ +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow-kubernetes +spec: + image: + productVersion: "{{ test_scenario['values']['airflow-latest'] }}" + clusterConfig: + loadExamples: true + exposeConfig: false + listenerClass: external-unstable + credentialsSecret: airflow-credentials + webservers: + roleGroups: + default: + replicas: 1 + config: + resources: + cpu: + min: 410m + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 810m + config: + resources: + cpu: + min: 400m + max: 800m + memory: + limit: 2Gi + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 750m + requests: + cpu: 350m + kubernetesExecutors: + config: + resources: + cpu: + min: 400m + max: 800m + memory: + limit: 2Gi + podOverrides: + spec: + containers: + - name: base + resources: + limits: + cpu: 750m + envOverrides: + ROLE_VAR: role-value # there are no role groups for kubernetes executors + AIRFLOW__METRICS__STATSD_ON: "False" # also set by the operator + schedulers: + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/overrides/21-assert.yaml b/tests/templates/kuttl/overrides/21-assert.yaml new file mode 100644 index 00000000..5717d5c5 --- /dev/null +++ b/tests/templates/kuttl/overrides/21-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +commands: +- script: | + kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers.[0].resources.limits | select (.cpu == "750m")' + kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers[] | select (.name == "base") | .env[] | select (.name == "AIRFLOW__METRICS__STATSD_ON" and .value == "False")' diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index df01a4c9..367467dc 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -69,6 +69,10 @@ tests: dimensions: - airflow-latest - openshift + - name: overrides + dimensions: + - airflow-latest + - openshift suites: - name: nightly # Run nightly with the latest airflow From e8cdf10916d7247476b4d516b3b0c07fbabf6911 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:30:55 +0200 Subject: [PATCH 2/4] Allow arbitrary python code (#493) * Bumping operator-rs to 0.74.0 * Adding docs * Allow arbitrary python code with Footer and Header keys * Update Changelog * Adding some tests for new file vars * Reworking tests, green now * Code Review Co-authored-by: Sebastian Bernauer --------- Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 5 ++++ .../airflow/pages/usage-guide/overrides.adoc | 27 +++++++++++++++++++ .../operator-binary/src/airflow_controller.rs | 24 ++++++++++++++++- .../smoke/40-install-airflow-cluster.yaml.j2 | 11 ++++++++ tests/templates/kuttl/smoke/41-assert.yaml | 11 ++++++++ 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 tests/templates/kuttl/smoke/41-assert.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d191d87..7473f58e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Allowing arbitrary python code as `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` in `webserver_config.py` ([#493]). + ### Changed - Reduce CRD size from `1.7MB` to `111KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#488]): @@ -16,6 +20,7 @@ [#488]: https://github.com/stackabletech/airflow-operator/pull/488 [#489]: https://github.com/stackabletech/airflow-operator/pull/489 +[#493]: https://github.com/stackabletech/airflow-operator/pull/493 ## [24.7.0] - 2024-07-24 diff --git a/docs/modules/airflow/pages/usage-guide/overrides.adoc b/docs/modules/airflow/pages/usage-guide/overrides.adoc index 5e082d2f..88e14b23 100644 --- a/docs/modules/airflow/pages/usage-guide/overrides.adoc +++ b/docs/modules/airflow/pages/usage-guide/overrides.adoc @@ -10,6 +10,33 @@ that each component has the same configuration: not all components use each sett Airflow exposes an environment variable for every Airflow configuration setting, a list of which can be found in the https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html[Configuration Reference]. +As Airflow can be configured with python code too, arbitrary code can be added to the `webserver_config.py`. +You can use either `EXPERIMENTAL_FILE_HEADER` to add code to the top or `EXPERIMENTAL_FILE_FOOTER` to add to the bottom. + +IMPORTANT: This is an experimental feature + +[source,yaml] +---- +webservers: + configOverrides: + webserver_config.py: + CSV_EXPORT: "{'encoding': 'utf-8'}" + EXPERIMENTAL_FILE_HEADER: | + from modules.my_module import my_class + EXPERIMENTAL_FILE_FOOTER: | + import logging + from airflow.security import AirflowSecurityManager + + class myCustomSecurityManger(AirflowSecurityManager): + def __init__(): + init() + + CUSTOM_SECURITY_MANAGER = myCustomSecurityManger + roleGroups: + default: + config: {} +---- + Although Kubernetes can override these settings in one of two ways (Configuration overrides, or Environment Variable overrides), the affect is the same and currently only the latter is implemented. This is described in the following section. diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 2ddee456..3c1a9721 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -50,7 +50,10 @@ use stackable_operator::{ }, kvp::{Label, LabelError, Labels}, logging::controller::ReconcilerError, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, + product_config_utils::{ + transform_all_roles_to_config, validate_all_roles_and_groups_config, + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, + }, product_logging::{ self, spec::{ContainerLogConfig, Logging}, @@ -65,6 +68,7 @@ use stackable_operator::{ }; use std::{ collections::{BTreeMap, HashMap}, + io::Write, str::FromStr, sync::Arc, }; @@ -280,6 +284,11 @@ pub enum Error { #[snafu(display("failed to construct config"))] ConstructConfig { source: config::Error }, + + #[snafu(display( + "failed to write to String (Vec to be precise) containing Airflow config" + ))] + WriteToConfigFileString { source: std::io::Error }, } type Result = std::result::Result; @@ -639,6 +648,15 @@ fn build_rolegroup_config_map( config::add_airflow_config(&mut config, authentication_config).context(ConstructConfigSnafu)?; let mut config_file = Vec::new(); + + // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; + } + + let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + flask_app_config_writer::write::( &mut config_file, config.iter(), @@ -648,6 +666,10 @@ fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; + } + let mut cm_builder = ConfigMapBuilder::new(); cm_builder diff --git a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 index c24abd8c..f9dccdbb 100644 --- a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 @@ -45,9 +45,20 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + webserver_config.py: + EXPERIMENTAL_FILE_HEADER: | + COMMON_HEADER_VAR = "role-value" + ROLE_HEADER_VAR = "role-value" + EXPERIMENTAL_FILE_FOOTER: | + ROLE_FOOTER_VAR = "role-value" roleGroups: default: replicas: 1 + configOverrides: + webserver_config.py: + EXPERIMENTAL_FILE_HEADER: | + COMMON_HEADER_VAR = "group-value" {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: config: diff --git a/tests/templates/kuttl/smoke/41-assert.yaml b/tests/templates/kuttl/smoke/41-assert.yaml new file mode 100644 index 00000000..68389607 --- /dev/null +++ b/tests/templates/kuttl/smoke/41-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # + # Test envOverrides + # + - script: | + kubectl -n $NAMESPACE get cm airflow-webserver-default -o yaml | yq -e '.data."webserver_config.py"' | grep "COMMON_HEADER_VAR = \"group-value\"" + kubectl -n $NAMESPACE get cm airflow-webserver-default -o yaml | yq -e '.data."webserver_config.py"' | grep "ROLE_FOOTER_VAR = \"role-value\"" From f21347393bedf9313aea1b6b2f116e4f4f1eb06c Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:05:58 +0200 Subject: [PATCH 3/4] Fix/more readable smoke test assert (#496) * Updating tests for more readability and better code style * Adopting to airflow tests --- tests/templates/kuttl/smoke/41-assert.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/smoke/41-assert.yaml b/tests/templates/kuttl/smoke/41-assert.yaml index 68389607..f099e0a6 100644 --- a/tests/templates/kuttl/smoke/41-assert.yaml +++ b/tests/templates/kuttl/smoke/41-assert.yaml @@ -7,5 +7,16 @@ commands: # Test envOverrides # - script: | - kubectl -n $NAMESPACE get cm airflow-webserver-default -o yaml | yq -e '.data."webserver_config.py"' | grep "COMMON_HEADER_VAR = \"group-value\"" - kubectl -n $NAMESPACE get cm airflow-webserver-default -o yaml | yq -e '.data."webserver_config.py"' | grep "ROLE_FOOTER_VAR = \"role-value\"" + #!/usr/bin/env bash + set -euo pipefail + + # Config Test Data + AIRFLOW_CONFIG=$( + kubectl -n "$NAMESPACE" get cm airflow-webserver-default -o yaml \ + | yq -e '.data["webserver_config.py"]' + ) + + # Config Test Assertions + echo "$AIRFLOW_CONFIG" | grep 'COMMON_HEADER_VAR = "group-value"' + echo "$AIRFLOW_CONFIG" | grep 'ROLE_FOOTER_VAR = "role-value"' + echo "$AIRFLOW_CONFIG" | grep -v 'ROLE_HEADER_VAR = "role-value"' From e3b254f08aa9a8b9a8f0b3b9f3316ac034d8c9f7 Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Tue, 3 Sep 2024 20:17:28 +1200 Subject: [PATCH 4/4] test(kuttl): remove -o pipefail bash option (#498) --- tests/templates/kuttl/smoke/41-assert.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/templates/kuttl/smoke/41-assert.yaml b/tests/templates/kuttl/smoke/41-assert.yaml index f099e0a6..6169fd93 100644 --- a/tests/templates/kuttl/smoke/41-assert.yaml +++ b/tests/templates/kuttl/smoke/41-assert.yaml @@ -7,8 +7,7 @@ commands: # Test envOverrides # - script: | - #!/usr/bin/env bash - set -euo pipefail + set -eu # Config Test Data AIRFLOW_CONFIG=$(