From 923dbd1b84c57a69859a026590f9691c062446e5 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 14 Jul 2023 08:53:27 +0000 Subject: [PATCH] Increase log volume size limit (#259) # Description *Please add a description here. This will become the commit message of the merge request later.* Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 4 +- Cargo.lock | 156 +++++++++--------- Cargo.toml | 2 +- Tiltfile | 2 +- rust/crd/src/constants.rs | 21 ++- rust/crd/src/lib.rs | 4 +- .../operator-binary/src/history_controller.rs | 9 +- rust/operator-binary/src/product_logging.rs | 8 +- 8 files changed, 113 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c6e6695..d901f7ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ All notable changes to this project will be documented in this file. ### Changed -- `operator-rs` `0.38.0` -> `0.41.0` ([#235]). +- `operator-rs` `0.38.0` -> `0.44.0` ([#235], [#259]). - Use 0.0.0-dev product images for testing ([#236]) - Use testing-tools 0.2.0 ([#236]) - Run as root group ([#241]). @@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file. ### Fixed - Fix quoting issues when spark config values contain spaces ([#243]). +- Increase the size limit of log volumes (#[259]) [#235]: https://github.com/stackabletech/spark-k8s-operator/pull/235 [#236]: https://github.com/stackabletech/spark-k8s-operator/pull/236 @@ -34,6 +35,7 @@ All notable changes to this project will be documented in this file. [#252]: https://github.com/stackabletech/spark-k8s-operator/pull/252 [#249]: https://github.com/stackabletech/spark-k8s-operator/pull/249 [#256]: https://github.com/stackabletech/spark-k8s-operator/pull/256 +[#259]: https://github.com/stackabletech/spark-k8s-operator/pull/259 ## [23.4.0] - 2023-04-17 diff --git a/Cargo.lock b/Cargo.lock index 66ff718c..78fbb7c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,13 +110,13 @@ checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "async-trait" -version = "0.1.70" +version = "0.1.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79fa67157abdfd688a259b6648808757db9347af834624f27ec646da976aee5d" +checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -267,9 +267,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.10" +version = "4.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a" +checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" dependencies = [ "clap_builder", "clap_derive", @@ -278,9 +278,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.10" +version = "4.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d" +checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" dependencies = [ "anstream", "anstyle", @@ -298,7 +298,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -370,12 +370,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core 0.20.1", - "darling_macro 0.20.1", + "darling_core 0.20.3", + "darling_macro 0.20.3", ] [[package]] @@ -394,16 +394,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -419,23 +419,23 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core 0.20.1", + "darling_core 0.20.3", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.0", "lock_api", "once_cell", "parking_lot_core", @@ -557,9 +557,9 @@ checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" [[package]] name = "equivalent" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -678,7 +678,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -940,9 +940,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "is-terminal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", "rustix", @@ -1307,7 +1307,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -1466,7 +1466,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -1495,9 +1495,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] @@ -1588,14 +1588,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89089e897c013b3deb627116ae56a6955a72b8bed395c9526af31c9fe528b484" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.0", - "regex-syntax 0.7.3", + "regex-automata 0.3.3", + "regex-syntax 0.7.4", ] [[package]] @@ -1609,13 +1609,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa250384981ea14565685dea16a9ccc4d1c541a13f82b9c168572264d1df8c56" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.3", + "regex-syntax 0.7.4", ] [[package]] @@ -1626,9 +1626,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rstest" @@ -1673,9 +1673,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.2" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ "bitflags 2.3.3", "errno", @@ -1747,9 +1747,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.166" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] @@ -1766,13 +1766,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.166" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -1788,9 +1788,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c" +checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" dependencies = [ "indexmap 2.0.0", "itoa", @@ -1861,15 +1861,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "snafu" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", "snafu-derive", @@ -1877,9 +1877,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck", "proc-macro2", @@ -1899,8 +1899,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.43.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.43.0#5d2ca16a6136d3994bf89a6bd5aa3a2180711251" +version = "0.44.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.44.0#f922369d05e709c0965098012d0fd76865004982" dependencies = [ "chrono", "clap", @@ -1933,13 +1933,13 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.43.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.43.0#5d2ca16a6136d3994bf89a6bd5aa3a2180711251" +version = "0.44.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.44.0#f922369d05e709c0965098012d0fd76865004982" dependencies = [ - "darling 0.20.1", + "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -2019,9 +2019,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.23" +version = "2.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" dependencies = [ "proc-macro2", "quote", @@ -2030,22 +2030,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.41" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c16a64ba9387ef3fdae4f9c1a7f07a0997fce91985c0336f1ddc1822b3b37802" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.41" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d14928354b01c4d6a4f0e549069adef399a284e7995c7ccca94e8a07a5346c59" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -2131,7 +2131,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -2174,9 +2174,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" dependencies = [ "serde", "serde_spanned", @@ -2195,9 +2195,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.11" +version = "0.19.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7" +checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" dependencies = [ "indexmap 2.0.0", "serde", @@ -2277,7 +2277,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -2470,7 +2470,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", "wasm-bindgen-shared", ] @@ -2492,7 +2492,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2602,9 +2602,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448" +checksum = "81a2094c43cc94775293eaa0e499fbc30048a6d824ac82c0351a8c0bf9112529" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index a2b5b09c..8cda3367 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [ ] [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.43.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.44.0" } semver = "1.0" serde = "1.0" serde_json = "1.0" diff --git a/Tiltfile b/Tiltfile index 4d9cc0e8..d7518408 100644 --- a/Tiltfile +++ b/Tiltfile @@ -35,4 +35,4 @@ helm_crds, helm_non_crds = filter_yaml( api_version = "^apiextensions\\.k8s\\.io/.*$", kind = "^CustomResourceDefinition$", ) -k8s_yaml(helm_non_crds) +k8s_yaml(helm_non_crds) \ No newline at end of file diff --git a/rust/crd/src/constants.rs b/rust/crd/src/constants.rs index 1460e3b3..bd677a35 100644 --- a/rust/crd/src/constants.rs +++ b/rust/crd/src/constants.rs @@ -1,3 +1,5 @@ +use stackable_operator::memory::{BinaryMultiple, MemoryQuantity}; + pub const APP_NAME: &str = "spark-k8s"; pub const VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES: &str = "driver-pod-template"; @@ -40,10 +42,21 @@ pub const MIN_MEMORY_OVERHEAD: u32 = 384; pub const JVM_OVERHEAD_FACTOR: f32 = 0.1; pub const NON_JVM_OVERHEAD_FACTOR: f32 = 0.4; -pub const MAX_SPARK_LOG_FILES_SIZE_IN_MIB: u32 = 10; -pub const MAX_INIT_CONTAINER_LOG_FILES_SIZE_IN_MIB: u32 = 1; -pub const LOG_VOLUME_SIZE_IN_MIB: u32 = - MAX_SPARK_LOG_FILES_SIZE_IN_MIB + MAX_INIT_CONTAINER_LOG_FILES_SIZE_IN_MIB; +pub const MAX_SPARK_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +// Spark applications can have up to three init containers. +// * `job` - copies user's application jar file from a volume to the expected location. +// * `requirements` - install Python packages. +// * `tls` - Generates a java trust store. +// All of these containers have their output captured by Vector via `capture_shell_output` +// and therefore also need space on the logging volume. +pub const MAX_INIT_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 3.0, // 1Mib for each possible container. + unit: BinaryMultiple::Mebi, +}; pub const OPERATOR_NAME: &str = "spark.stackable.tech"; pub const CONTROLLER_NAME: &str = "sparkapplication"; diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 06cb40d5..17ca743e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -324,7 +324,9 @@ impl SparkApplication { VolumeBuilder::new(VOLUME_MOUNT_NAME_LOG) .with_empty_dir( None::, - Some(Quantity(format!("{LOG_VOLUME_SIZE_IN_MIB}Mi"))), + Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_SPARK_LOG_FILES_SIZE, MAX_INIT_LOG_FILES_SIZE], + )), ) .build(), ); diff --git a/rust/operator-binary/src/history_controller.rs b/rust/operator-binary/src/history_controller.rs index f7444c41..c9b0c613 100644 --- a/rust/operator-binary/src/history_controller.rs +++ b/rust/operator-binary/src/history_controller.rs @@ -1,3 +1,4 @@ +use crate::product_logging::{self, resolve_vector_aggregator_address}; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, VolumeBuilder}, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -11,7 +12,7 @@ use stackable_operator::{ }, rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, - apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, + apimachinery::pkg::apis::meta::v1::LabelSelector, }, kube::{ runtime::{controller::Action, reflector::ObjectRef}, @@ -20,7 +21,7 @@ use stackable_operator::{ labels::{role_group_selector_labels, role_selector_labels, ObjectLabels}, product_config::ProductConfigManager, product_logging::{ - framework::vector_container, + framework::{calculate_log_volume_size_limit, vector_container}, spec::{ ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, CustomContainerLogConfig, @@ -44,8 +45,6 @@ use stackable_operator::k8s_openapi::DeepMerge; use stackable_operator::logging::controller::ReconcilerError; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::product_logging::{self, resolve_vector_aggregator_address}; - pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, @@ -339,7 +338,7 @@ fn build_stateful_set( VolumeBuilder::new(VOLUME_MOUNT_NAME_LOG) .with_empty_dir( None::, - Some(Quantity(format!("{LOG_VOLUME_SIZE_IN_MIB}Mi"))), + Some(calculate_log_volume_size_limit(&[MAX_SPARK_LOG_FILES_SIZE])), ) .build(), ) diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index cfe244d5..896a1dae 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -6,6 +6,7 @@ use stackable_operator::{ client::Client, k8s_openapi::api::core::v1::ConfigMap, kube::Resource, + memory::BinaryMultiple, product_logging::{ self, spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, @@ -13,7 +14,7 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; use stackable_spark_k8s_crd::constants::{ - LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE_IN_MIB, VOLUME_MOUNT_PATH_LOG, + LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE, VOLUME_MOUNT_PATH_LOG, }; #[derive(Snafu, Debug)] @@ -90,7 +91,10 @@ where product_logging::framework::create_log4j2_config( &format!("{VOLUME_MOUNT_PATH_LOG}/{main_container}"), LOG_FILE, - MAX_SPARK_LOG_FILES_SIZE_IN_MIB, + MAX_SPARK_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, CONSOLE_CONVERSION_PATTERN, log_config, ),