diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e85de40..81c865fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Allowing arbitrary python code as `EXPERIMENTAL_FILE_HEADER` and `EXPERIMENTAL_FILE_FOOTER` in `webserver_config.py` ([#493]). - Support for `2.9.3` ([#494]). ### Changed @@ -21,6 +22,7 @@ [#488]: https://github.com/stackabletech/airflow-operator/pull/488 [#489]: https://github.com/stackabletech/airflow-operator/pull/489 +[#493]: https://github.com/stackabletech/airflow-operator/pull/493 [#494]: https://github.com/stackabletech/airflow-operator/pull/494 ## [24.7.0] - 2024-07-24 diff --git a/Cargo.lock b/Cargo.lock index 112b680b..960a6ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -157,7 +157,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -270,12 +270,13 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -299,9 +300,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.15" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -328,7 +329,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -390,9 +391,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -443,7 +444,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -454,7 +455,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -465,7 +466,7 @@ checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -672,7 +673,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -991,9 +992,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown", @@ -1048,9 +1049,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -1185,7 +1186,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1224,9 +1225,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libgit2-sys" @@ -1242,9 +1243,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" dependencies = [ "cc", "libc", @@ -1524,7 +1525,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1555,7 +1556,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1764,7 +1765,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.74", + "syn 2.0.75", "unicode-ident", ] @@ -1800,9 +1801,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -1881,7 +1882,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1931,9 +1932,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.206" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -1950,13 +1951,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.206" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1967,14 +1968,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] name = "serde_json" -version = "1.0.124" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "itoa", "memchr", @@ -2026,6 +2027,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -2089,7 +2096,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2145,8 +2152,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.73.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +version = "0.74.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "chrono", "clap", @@ -2159,7 +2166,6 @@ dependencies = [ "json-patch", "k8s-openapi", "kube", - "lazy_static", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2183,12 +2189,12 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2216,7 +2222,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2238,9 +2244,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.74" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -2264,7 +2270,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2347,9 +2353,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", @@ -2371,7 +2377,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2465,15 +2471,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -2507,7 +2513,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2621,9 +2627,9 @@ dependencies = [ [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" [[package]] name = "unsafe-libyaml" @@ -2690,34 +2696,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2725,22 +2732,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-time" @@ -2889,7 +2896,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] diff --git a/Cargo.nix b/Cargo.nix index 43d079b2..b83c84f2 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -446,7 +446,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "visit-mut" ]; } ]; @@ -473,7 +473,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "full" "visit-mut" "parsing" "printing" "proc-macro" ]; } @@ -776,9 +776,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.1.10"; + version = "1.1.13"; edition = "2018"; - sha256 = "14jjp993s17yfvl5linq3gs4jzb7f42nkq09r5kyfjskmjzsms79"; + sha256 = "0j6czmvfj9qhqmazznvg2wkzlyh7p65pwdqhnl1rpvpc8xwjznvj"; authors = [ "Alex Crichton " ]; @@ -796,6 +796,10 @@ rec { usesDefaultFeatures = false; target = { target, features }: (target."unix" or false); } + { + name = "shlex"; + packageId = "shlex"; + } ]; features = { "parallel" = [ "dep:libc" "dep:jobserver" ]; @@ -880,10 +884,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.5.15"; + version = "4.5.16"; edition = "2021"; crateBin = []; - sha256 = "1k656mr99srcla2bx2h7wjwlb9mywavb5k1szpd5b9pxaj287n0i"; + sha256 = "068hjwbrndn4iz4fsc6d52q4ymg1kfsymjnqbxzdil23zbzijrzd"; dependencies = [ { name = "clap_builder"; @@ -979,7 +983,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -1139,9 +1143,9 @@ rec { }; "cpufeatures" = rec { crateName = "cpufeatures"; - version = "0.2.12"; + version = "0.2.13"; edition = "2018"; - sha256 = "012m7rrak4girqlii3jnqwrr73gv1i980q4wra5yyyhvzwk5xzjk"; + sha256 = "1b89kljf7phyh63vxwsvf9lbgwkv0dsj7pcjmqgysnwsvkk55s2i"; authors = [ "RustCrypto Developers" ]; @@ -1283,7 +1287,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "extra-traits" ]; } ]; @@ -1313,7 +1317,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; @@ -1339,7 +1343,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "visit-mut" ]; } ]; @@ -1884,7 +1888,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -2945,9 +2949,9 @@ rec { }; "indexmap" = rec { crateName = "indexmap"; - version = "2.3.0"; + version = "2.4.0"; edition = "2021"; - sha256 = "182yvyck0zk3swa9bs8l9a8lmis3q7zy338wj6rx2bd81giw4gyy"; + sha256 = "0p2hwvmir50qcl5q6lib8fjq5dzv4f0gqy8czcyfva3yzhzdbslk"; dependencies = [ { name = "equivalent"; @@ -3076,9 +3080,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.69"; - edition = "2018"; - sha256 = "0v99rz97asnzapb0jsc3jjhvxpfxr7h7qd97yqyrf9i7viimbh99"; + version = "0.3.70"; + edition = "2021"; + sha256 = "0yp3rz7vrn9mmqdpkds426r1p9vs6i8mkxx8ryqdfadr0s2q0s0q"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -3678,7 +3682,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "extra-traits" ]; } ]; @@ -3820,9 +3824,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.155"; + version = "0.2.158"; edition = "2015"; - sha256 = "0z44c53z54znna8n322k5iwg80arxxpdzjj5260pxxzc9a58icwp"; + sha256 = "0fb4qldw1jrxljrwz6bsjn8lv4rqizlqmab41q3j98q332xw9bfq"; authors = [ "The Rust Project Developers" ]; @@ -3880,10 +3884,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.18"; + version = "1.1.19"; edition = "2018"; links = "z"; - sha256 = "0bpqmfzvijbrqs29vphnafjz834lpz6pabbsnf85rqppb9pa4pf1"; + sha256 = "0is6lzp9xrgryn57d6ajvadkrs40vzvk37yjkyy6wjd7k5vkmigx"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -4773,7 +4777,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; features = { @@ -4846,7 +4850,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "visit-mut" ]; } ]; @@ -5483,7 +5487,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ]; } { @@ -5605,9 +5609,9 @@ rec { }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; - version = "0.7.1"; + version = "0.7.2"; edition = "2021"; - sha256 = "1fjzn7h3p6f3wr014p0i58131513b5li3146v6yrdd2i0r16v3d8"; + sha256 = "1ynz8a108bd3k7zix26z79n8cnqfsrfflsf01x7406h9r7zjs604"; libName = "rustls_native_certs"; dependencies = [ { @@ -5848,7 +5852,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "extra-traits" ]; } ]; @@ -5988,9 +5992,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.206"; + version = "1.0.208"; edition = "2018"; - sha256 = "1102kmaf7cdln5f44l1ygqz457ly9pii3kcb28555p9387clqgjv"; + sha256 = "1cng2zkvv6hh137jrrqdmhi2hllfnqwlqjgviqjalkv8rg98bw6g"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6043,9 +6047,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.206"; + version = "1.0.208"; edition = "2015"; - sha256 = "15rgfrcs56hqhhfnhq4413sazcfzkkvcr66g121fm0r3il9vdgzs"; + sha256 = "1krblvy84j6d9zj12ms9l2wfbwf2w8jazkx0bf7fs4vnzy0qw014"; procMacro = true; authors = [ "Erick Tryzelaar " @@ -6066,7 +6070,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -6098,7 +6102,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -6107,9 +6111,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.124"; + version = "1.0.125"; edition = "2021"; - sha256 = "13g3zka165qqnvbw0c6k708wdfc4738sr4f8ijjxpcsnga265bb6"; + sha256 = "1v9idlv5nq5f000qsv6v02h2zbsfk2m6d05af2zgbk3kl0sygj43"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6279,6 +6283,24 @@ rec { "loom" = [ "dep:loom" ]; }; }; + "shlex" = rec { + crateName = "shlex"; + version = "1.3.0"; + edition = "2015"; + sha256 = "0r1y6bv26c1scpxvhg2cabimrmwgbp4p3wy6syj9n0c4s3q2znhg"; + authors = [ + "comex " + "Fenhl " + "Adrian Taylor " + "Alex Touchet " + "Daniel Parks " + "Garrett Berg " + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "signal-hook-registry" = rec { crateName = "signal-hook-registry"; version = "1.4.2"; @@ -6451,7 +6473,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -6653,13 +6675,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.73.0"; + version = "0.74.0"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "4d98a29b08a7d959e5e287f774cf064c02ffbd62"; - sha256 = "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m"; + rev = "c77a5423b66bc1667b63af7d8bec00de88a5303f"; + sha256 = "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya"; }; libName = "stackable_operator"; authors = [ @@ -6716,10 +6738,6 @@ rec { usesDefaultFeatures = false; features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; } - { - name = "lazy_static"; - packageId = "lazy_static"; - } { name = "opentelemetry-jaeger"; packageId = "opentelemetry-jaeger"; @@ -6812,8 +6830,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "4d98a29b08a7d959e5e287f774cf064c02ffbd62"; - sha256 = "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m"; + rev = "c77a5423b66bc1667b63af7d8bec00de88a5303f"; + sha256 = "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -6835,7 +6853,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; @@ -6908,7 +6926,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "parsing" "extra-traits" ]; } ]; @@ -6961,11 +6979,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit" ]; }; - "syn 2.0.74" = rec { + "syn 2.0.75" = rec { crateName = "syn"; - version = "2.0.74"; + version = "2.0.75"; edition = "2021"; - sha256 = "1mxk5h27mzl0r2j434rqrm279k302jdl1lr16kc0nval7lgb9khz"; + sha256 = "1nf72xc8pabwny8d0d4kjk5vmadg3fa240dywznka6gw6hq0dbzn"; authors = [ "David Tolnay " ]; @@ -7031,7 +7049,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; @@ -7270,9 +7288,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.39.2"; + version = "1.39.3"; edition = "2021"; - sha256 = "1cb7yhba7nnf00cylcywk7rai5kkdb8b4jzwrc26zgbqqwdzp96s"; + sha256 = "1xgzhj7bxqqpjaabjkgsx8hi0f600bzj4iyp9f0a9gr3k6dwkawv"; authors = [ "Tokio Contributors " ]; @@ -7393,7 +7411,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } ]; @@ -7801,9 +7819,9 @@ rec { }; "tower-layer" = rec { crateName = "tower-layer"; - version = "0.3.2"; + version = "0.3.3"; edition = "2018"; - sha256 = "1l7i17k9vlssrdg4s3b0ia5jjkmmxsvv8s9y9ih0jfi8ssz8s362"; + sha256 = "03kq92fdzxin51w8iqix06dcfgydyvx7yr6izjq0p626v9n2l70j"; libName = "tower_layer"; authors = [ "Tower Maintainers " @@ -7812,9 +7830,9 @@ rec { }; "tower-service" = rec { crateName = "tower-service"; - version = "0.3.2"; + version = "0.3.3"; edition = "2018"; - sha256 = "0lmfzmmvid2yp2l36mbavhmqgsvzqf7r2wiwz73ml4xmwaf1rg5n"; + sha256 = "1hzfkvkci33ra94xjx64vv3pp0sq346w06fpkcdwjcid7zhvdycd"; libName = "tower_service"; authors = [ "Tower Maintainers " @@ -7927,7 +7945,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -8344,9 +8362,9 @@ rec { }; "unicode-xid" = rec { crateName = "unicode-xid"; - version = "0.2.4"; + version = "0.2.5"; edition = "2015"; - sha256 = "131dfzf7d8fsr1ivch34x42c2d1ik5ig3g78brxncnn0r1sdyqpr"; + sha256 = "02n96yhcjwx7vphbwf9p7xfqbwvqgzdw2qz4h0x3wd5wgxj315r2"; libName = "unicode_xid"; authors = [ "erick.tryzelaar " @@ -8500,9 +8518,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1a4mcw13nsk3fr8fxjzf9kk1wj88xkfsmnm0pjraw01ryqfm7qjb"; + version = "0.2.93"; + edition = "2021"; + sha256 = "1dfr7pka5kwvky2fx82m9d060p842hc5fyyw8igryikcdb0xybm8"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -8512,6 +8530,10 @@ rec { name = "cfg-if"; packageId = "cfg-if"; } + { + name = "once_cell"; + packageId = "once_cell"; + } { name = "wasm-bindgen-macro"; packageId = "wasm-bindgen-macro"; @@ -8520,7 +8542,6 @@ rec { features = { "default" = [ "spans" "std" ]; "enable-interning" = [ "std" ]; - "gg-alloc" = [ "wasm-bindgen-test/gg-alloc" ]; "serde" = [ "dep:serde" ]; "serde-serialize" = [ "serde" "serde_json" "std" ]; "serde_json" = [ "dep:serde_json" ]; @@ -8532,9 +8553,9 @@ rec { }; "wasm-bindgen-backend" = rec { crateName = "wasm-bindgen-backend"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1nj7wxbi49f0rw9d44rjzms26xlw6r76b2mrggx8jfbdjrxphkb1"; + version = "0.2.93"; + edition = "2021"; + sha256 = "0yypblaf94rdgqs5xw97499xfwgs1096yx026d6h88v563d9dqwx"; libName = "wasm_bindgen_backend"; authors = [ "The wasm-bindgen Developers" @@ -8562,7 +8583,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "full" ]; } { @@ -8577,9 +8598,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.92"; - edition = "2018"; - sha256 = "09npa1srjjabd6nfph5yc03jb26sycjlxhy0c2a1pdrpx4yq5y51"; + version = "0.2.93"; + edition = "2021"; + sha256 = "1kycd1xfx4d9xzqknvzbiqhwb5fzvjqrrn88x692q1vblj8lqp2q"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -8603,9 +8624,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.92"; - edition = "2018"; - sha256 = "1dqv2xs8zcyw4kjgzj84bknp2h76phmsb3n7j6hn396h4ssifkz9"; + version = "0.2.93"; + edition = "2021"; + sha256 = "0dp8w6jmw44srym6l752nkr3hkplyw38a2fxz5f3j1ch9p3l1hxg"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -8621,7 +8642,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; features = [ "visit" "full" ]; } { @@ -8641,10 +8662,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.92"; - edition = "2018"; + version = "0.2.93"; + edition = "2021"; links = "wasm_bindgen"; - sha256 = "15kyavsrna2cvy30kg03va257fraf9x00ny554vxngvpyaa0q6dg"; + sha256 = "1104bny0hv40jfap3hp8jhs0q4ya244qcrvql39i38xlghq0lan6"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -9218,7 +9239,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.74"; + packageId = "syn 2.0.75"; } ]; diff --git a/Cargo.toml b/Cargo.toml index 190cef64..9a5a8100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.39", features = ["full"] } tracing = "0.1" diff --git a/crate-hashes.json b/crate-hashes.json index ddd91d81..8725a492 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,5 +1,5 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#stackable-operator-derive@0.3.1": "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#stackable-operator@0.73.0": "0cmfbc3v9kklsfkqbnhwig45106gfizhmlmg9p1qgdjp8az43l9m", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#stackable-operator-derive@0.3.1": "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#stackable-operator@0.74.0": "1g1a0v98wlcb36ibwv1nv75g3b3s1mjmaps443fc2w2maam94lya", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/docs/modules/airflow/pages/usage-guide/overrides.adoc b/docs/modules/airflow/pages/usage-guide/overrides.adoc index 5e082d2f..88e14b23 100644 --- a/docs/modules/airflow/pages/usage-guide/overrides.adoc +++ b/docs/modules/airflow/pages/usage-guide/overrides.adoc @@ -10,6 +10,33 @@ that each component has the same configuration: not all components use each sett Airflow exposes an environment variable for every Airflow configuration setting, a list of which can be found in the https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html[Configuration Reference]. +As Airflow can be configured with python code too, arbitrary code can be added to the `webserver_config.py`. +You can use either `EXPERIMENTAL_FILE_HEADER` to add code to the top or `EXPERIMENTAL_FILE_FOOTER` to add to the bottom. + +IMPORTANT: This is an experimental feature + +[source,yaml] +---- +webservers: + configOverrides: + webserver_config.py: + CSV_EXPORT: "{'encoding': 'utf-8'}" + EXPERIMENTAL_FILE_HEADER: | + from modules.my_module import my_class + EXPERIMENTAL_FILE_FOOTER: | + import logging + from airflow.security import AirflowSecurityManager + + class myCustomSecurityManger(AirflowSecurityManager): + def __init__(): + init() + + CUSTOM_SECURITY_MANAGER = myCustomSecurityManger + roleGroups: + default: + config: {} +---- + Although Kubernetes can override these settings in one of two ways (Configuration overrides, or Environment Variable overrides), the affect is the same and currently only the latter is implemented. This is described in the following section. diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 41f7da30..81d74c03 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -50,7 +50,10 @@ use stackable_operator::{ }, kvp::{Label, LabelError, Labels}, logging::controller::ReconcilerError, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, + product_config_utils::{ + transform_all_roles_to_config, validate_all_roles_and_groups_config, + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, + }, product_logging::{ self, spec::{ContainerLogConfig, Logging}, @@ -65,6 +68,7 @@ use stackable_operator::{ }; use std::{ collections::{BTreeMap, HashMap}, + io::Write, str::FromStr, sync::Arc, }; @@ -280,6 +284,11 @@ pub enum Error { #[snafu(display("failed to construct config"))] ConstructConfig { source: config::Error }, + + #[snafu(display( + "failed to write to String (Vec to be precise) containing Airflow config" + ))] + WriteToConfigFileString { source: std::io::Error }, } type Result = std::result::Result; @@ -639,6 +648,15 @@ fn build_rolegroup_config_map( config::add_airflow_config(&mut config, authentication_config).context(ConstructConfigSnafu)?; let mut config_file = Vec::new(); + + // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; + } + + let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + flask_app_config_writer::write::( &mut config_file, config.iter(), @@ -648,6 +666,10 @@ fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; + } + let mut cm_builder = ConfigMapBuilder::new(); cm_builder diff --git a/tests/templates/kuttl/overrides/00-assert.yaml.j2 b/tests/templates/kuttl/overrides/00-assert.yaml.j2 new file mode 100644 index 00000000..4b8ea1bb --- /dev/null +++ b/tests/templates/kuttl/overrides/00-assert.yaml.j2 @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-master +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-replicas +status: + availableReplicas: 1 \ No newline at end of file diff --git a/tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 b/tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 new file mode 100644 index 00000000..f98f9be8 --- /dev/null +++ b/tests/templates/kuttl/overrides/00-install-dbs.yaml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-postgresql + --namespace $NAMESPACE + --version 12.1.5 + --set auth.username=airflow + --set auth.password=airflow + --set auth.database=airflow + --repo https://charts.bitnami.com/bitnami postgresql + timeout: 240 + - script: >- + helm install airflow-redis + --namespace $NAMESPACE + --version 17.3.7 + --set auth.password=redis + --set replica.replicaCount=1 + --repo https://charts.bitnami.com/bitnami redis + timeout: 240 \ No newline at end of file diff --git a/tests/templates/kuttl/overrides/10-assert.yaml b/tests/templates/kuttl/overrides/10-assert.yaml new file mode 100644 index 00000000..fb129c43 --- /dev/null +++ b/tests/templates/kuttl/overrides/10-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-celery --timeout 301s diff --git a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 new file mode 100644 index 00000000..2e7da7ff --- /dev/null +++ b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 @@ -0,0 +1,99 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-override +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow-celery +spec: + image: + productVersion: "{{ test_scenario['values']['airflow-latest'] }}" + clusterConfig: + loadExamples: true + exposeConfig: false + listenerClass: external-unstable + credentialsSecret: airflow-credentials + webservers: + envOverrides: + COMMON_VAR: role-value # overridden by role group below + ROLE_VAR: role-value # only defined here at role level + credentialsSecret: test-override + roleGroups: + default: + replicas: 1 + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + celeryExecutors: + envOverrides: + COMMON_VAR: role-value # overridden by role group below + ROLE_VAR: role-value # only defined here at role level + roleGroups: + default: + replicas: 2 + config: + resources: + cpu: + min: 410m + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 810m + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + config: + resources: + cpu: + min: 400m + max: 800m + memory: + limit: 2Gi + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 750m + requests: + cpu: 350m + schedulers: + roleGroups: + default: + replicas: 1 + envOverrides: + credentialsSecret: test-override diff --git a/tests/templates/kuttl/overrides/11-assert.yaml b/tests/templates/kuttl/overrides/11-assert.yaml new file mode 100644 index 00000000..33645a7a --- /dev/null +++ b/tests/templates/kuttl/overrides/11-assert.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # + # Test envOverrides + # + - script: | + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' + kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "credentialsSecret" and .value == "test-override")' + + + - script: | + kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' diff --git a/tests/templates/kuttl/overrides/20-assert.yaml b/tests/templates/kuttl/overrides/20-assert.yaml new file mode 100644 index 00000000..22025a66 --- /dev/null +++ b/tests/templates/kuttl/overrides/20-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-kubernetes --timeout 301s diff --git a/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 new file mode 100644 index 00000000..bbaf47b4 --- /dev/null +++ b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 @@ -0,0 +1,66 @@ +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow-kubernetes +spec: + image: + productVersion: "{{ test_scenario['values']['airflow-latest'] }}" + clusterConfig: + loadExamples: true + exposeConfig: false + listenerClass: external-unstable + credentialsSecret: airflow-credentials + webservers: + roleGroups: + default: + replicas: 1 + config: + resources: + cpu: + min: 410m + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 810m + config: + resources: + cpu: + min: 400m + max: 800m + memory: + limit: 2Gi + podOverrides: + spec: + containers: + - name: airflow + resources: + limits: + cpu: 750m + requests: + cpu: 350m + kubernetesExecutors: + config: + resources: + cpu: + min: 400m + max: 800m + memory: + limit: 2Gi + podOverrides: + spec: + containers: + - name: base + resources: + limits: + cpu: 750m + envOverrides: + ROLE_VAR: role-value # there are no role groups for kubernetes executors + AIRFLOW__METRICS__STATSD_ON: "False" # also set by the operator + schedulers: + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/overrides/21-assert.yaml b/tests/templates/kuttl/overrides/21-assert.yaml new file mode 100644 index 00000000..5717d5c5 --- /dev/null +++ b/tests/templates/kuttl/overrides/21-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +commands: +- script: | + kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers.[0].resources.limits | select (.cpu == "750m")' + kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | yq -e '.spec.containers[] | select (.name == "base") | .env[] | select (.name == "AIRFLOW__METRICS__STATSD_ON" and .value == "False")' diff --git a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 index c24abd8c..f9dccdbb 100644 --- a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 @@ -45,9 +45,20 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + webserver_config.py: + EXPERIMENTAL_FILE_HEADER: | + COMMON_HEADER_VAR = "role-value" + ROLE_HEADER_VAR = "role-value" + EXPERIMENTAL_FILE_FOOTER: | + ROLE_FOOTER_VAR = "role-value" roleGroups: default: replicas: 1 + configOverrides: + webserver_config.py: + EXPERIMENTAL_FILE_HEADER: | + COMMON_HEADER_VAR = "group-value" {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: config: diff --git a/tests/templates/kuttl/smoke/41-assert.yaml b/tests/templates/kuttl/smoke/41-assert.yaml new file mode 100644 index 00000000..6169fd93 --- /dev/null +++ b/tests/templates/kuttl/smoke/41-assert.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # + # Test envOverrides + # + - script: | + set -eu + + # Config Test Data + AIRFLOW_CONFIG=$( + kubectl -n "$NAMESPACE" get cm airflow-webserver-default -o yaml \ + | yq -e '.data["webserver_config.py"]' + ) + + # Config Test Assertions + echo "$AIRFLOW_CONFIG" | grep 'COMMON_HEADER_VAR = "group-value"' + echo "$AIRFLOW_CONFIG" | grep 'ROLE_FOOTER_VAR = "role-value"' + echo "$AIRFLOW_CONFIG" | grep -v 'ROLE_HEADER_VAR = "role-value"' diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 0d8dc9f3..ea4916ee 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -67,6 +67,10 @@ tests: dimensions: - airflow-latest - openshift + - name: overrides + dimensions: + - airflow-latest + - openshift suites: - name: nightly # Run nightly with the latest airflow