From 105c92d69ab762bf13a68b606e06d91cd751a3cf Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:39:52 +0200 Subject: [PATCH] First working version --- deploy/config-spec/properties.yaml | 39 ++++++++++- .../configs/properties.yaml | 39 ++++++++++- rust/crd/src/history.rs | 5 +- rust/crd/src/lib.rs | 23 ++++++- .../operator-binary/src/history_controller.rs | 64 ++++++++++++++----- 5 files changed, 148 insertions(+), 22 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index c8bdb8de..8bb23059 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -3,4 +3,41 @@ version: 0.1.0 spec: units: [] -properties: [] +properties: + - property: &jvmDnsCacheTtl + propertyNames: + - name: "networkaddress.cache.ttl" + kind: + type: "file" + file: "security.properties" + datatype: + type: "integer" + min: "0" + recommendedValues: + - fromVersion: "0.0.0" + value: "30" + roles: + - name: "node" + required: true + asOfVersion: "0.0.0" + comment: "History server - TTL for successfully resolved domain names." + description: "History server - TTL for successfully resolved domain names." + + - property: &jvmDnsCacheNegativeTtl + propertyNames: + - name: "networkaddress.cache.negative.ttl" + kind: + type: "file" + file: "security.properties" + datatype: + type: "integer" + min: "0" + recommendedValues: + - fromVersion: "0.0.0" + value: "0" + roles: + - name: "node" + required: true + asOfVersion: "0.0.0" + comment: "History server - TTL for domain names that cannot be resolved." + description: "History server - TTL for domain names that cannot be resolved." diff --git a/deploy/helm/spark-k8s-operator/configs/properties.yaml b/deploy/helm/spark-k8s-operator/configs/properties.yaml index c8bdb8de..8bb23059 100644 --- a/deploy/helm/spark-k8s-operator/configs/properties.yaml +++ b/deploy/helm/spark-k8s-operator/configs/properties.yaml @@ -3,4 +3,41 @@ version: 0.1.0 spec: units: [] -properties: [] +properties: + - property: &jvmDnsCacheTtl + propertyNames: + - name: "networkaddress.cache.ttl" + kind: + type: "file" + file: "security.properties" + datatype: + type: "integer" + min: "0" + recommendedValues: + - fromVersion: "0.0.0" + value: "30" + roles: + - name: "node" + required: true + asOfVersion: "0.0.0" + comment: "History server - TTL for successfully resolved domain names." + description: "History server - TTL for successfully resolved domain names." + + - property: &jvmDnsCacheNegativeTtl + propertyNames: + - name: "networkaddress.cache.negative.ttl" + kind: + type: "file" + file: "security.properties" + datatype: + type: "integer" + min: "0" + recommendedValues: + - fromVersion: "0.0.0" + value: "0" + roles: + - name: "node" + required: true + asOfVersion: "0.0.0" + comment: "History server - TTL for domain names that cannot be resolved." + description: "History server - TTL for domain names that cannot be resolved." diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index e708bfa1..28c72eb8 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -200,7 +200,10 @@ impl SparkHistoryServer { > = vec![( HISTORY_ROLE_NAME.to_string(), ( - vec![PropertyNameKind::File(SPARK_DEFAULTS_FILE_NAME.to_string())], + vec![ + PropertyNameKind::File(SPARK_DEFAULTS_FILE_NAME.to_string()), + PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), + ], self.spec.nodes.clone(), ), )] diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 4d4def4f..2f928ac1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -538,11 +538,28 @@ impl SparkApplication { } } - // s3 with TLS + // Extra JVM opts: + // - java security properties + // - s3 with TLS + let mut extra_java_opts = vec![format!( + "-Djava.security.properties={VOLUME_MOUNT_PATH_LOG_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" + )]; if tlscerts::tls_secret_names(s3conn, s3_log_dir).is_some() { - submit_cmd.push(format!("--conf spark.driver.extraJavaOptions=\"-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12 -Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD} -Djavax.net.ssl.trustStoreType=pkcs12 -Djavax.net.debug=ssl,handshake\"")); - submit_cmd.push(format!("--conf spark.executor.extraJavaOptions=\"-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12 -Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD} -Djavax.net.ssl.trustStoreType=pkcs12 -Djavax.net.debug=ssl,handshake\"")); + extra_java_opts.extend( + vec![ + format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12"), + format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}"), + format!("-Djavax.net.ssl.trustStoreType=pkcs12"), + format!("-Djavax.net.debug=ssl,handshake"), + ] + .into_iter(), + ); } + let str_extra_java_opts = extra_java_opts.join(" "); + submit_cmd.extend(vec![ + format!("--conf spark.driver.extraJavaOptions=\"{str_extra_java_opts}\""), + format!("--conf spark.executor.extraJavaOptions=\"{str_extra_java_opts}\""), + ]); // repositories and packages arguments if let Some(deps) = self.spec.deps.clone() { diff --git a/rust/operator-binary/src/history_controller.rs b/rust/operator-binary/src/history_controller.rs index 207c4cb2..fdd28424 100644 --- a/rust/operator-binary/src/history_controller.rs +++ b/rust/operator-binary/src/history_controller.rs @@ -19,7 +19,9 @@ use stackable_operator::{ Resource, ResourceExt, }, labels::{role_group_selector_labels, role_selector_labels, ObjectLabels}, - product_config::ProductConfigManager, + product_config::{ + types::PropertyNameKind, writer::to_java_properties_string, ProductConfigManager, + }, product_logging::{ framework::{calculate_log_volume_size_limit, vector_container}, spec::{ @@ -32,19 +34,20 @@ use stackable_operator::{ use stackable_spark_k8s_crd::{ constants::{ ACCESS_KEY_ID, APP_NAME, HISTORY_CONTROLLER_NAME, HISTORY_IMAGE_BASE_NAME, - HISTORY_ROLE_NAME, LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE, OPERATOR_NAME, - SECRET_ACCESS_KEY, SPARK_CLUSTER_ROLE, SPARK_DEFAULTS_FILE_NAME, SPARK_UID, - STACKABLE_TLS_STORE_PASSWORD, STACKABLE_TRUST_STORE, VOLUME_MOUNT_NAME_CONFIG, - VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_NAME_SPARK_DEFAULTS, - VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, VOLUME_MOUNT_PATH_SPARK_DEFAULTS, + HISTORY_ROLE_NAME, JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG_FILE, + MAX_SPARK_LOG_FILES_SIZE, OPERATOR_NAME, SECRET_ACCESS_KEY, SPARK_CLUSTER_ROLE, + SPARK_DEFAULTS_FILE_NAME, SPARK_UID, STACKABLE_TLS_STORE_PASSWORD, STACKABLE_TRUST_STORE, + VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, + VOLUME_MOUNT_NAME_SPARK_DEFAULTS, VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, + VOLUME_MOUNT_PATH_SPARK_DEFAULTS, }, history, history::{HistoryConfig, SparkHistoryServer, SparkHistoryServerContainer}, s3logdir::S3LogDir, tlscerts, }; -use std::time::Duration; use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::HashMap, time::Duration}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::builder::resources::ResourceRequirementsBuilder; @@ -129,6 +132,14 @@ pub enum Error { }, #[snafu(display("cannot retrieve role group"))] CannotRetrieveRoleGroup { source: history::Error }, + #[snafu(display( + "History server : failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for group {}", + rolegroup + ))] + JvmSecurityProperties { + source: stackable_operator::product_config::writer::PropertiesWriterError, + rolegroup: String, + }, } type Result = std::result::Result; @@ -211,7 +222,7 @@ pub async fn reconcile(shs: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, _error: &Error, _ctx: Arc>, + merged_config: &HistoryConfig, app_version_label: &str, rolegroupref: &RoleGroupRef, s3_log_dir: &S3LogDir, @@ -278,6 +291,16 @@ fn build_config_map( let spark_defaults = spark_defaults(shs, s3_log_dir, rolegroupref)?; + let jvm_sec_props: BTreeMap> = config + .get(&PropertyNameKind::File( + JVM_SECURITY_PROPERTIES_FILE.to_string(), + )) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect(); + let mut cm_builder = ConfigMapBuilder::new(); cm_builder @@ -290,12 +313,20 @@ fn build_config_map( .with_recommended_labels(labels(shs, app_version_label, &rolegroupref.role_group)) .build(), ) - .add_data(SPARK_DEFAULTS_FILE_NAME, spark_defaults); + .add_data(SPARK_DEFAULTS_FILE_NAME, spark_defaults) + .add_data( + JVM_SECURITY_PROPERTIES_FILE, + to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { + JvmSecurityPropertiesSnafu { + rolegroup: rolegroupref.role_group.clone(), + } + })?, + ); product_logging::extend_config_map( rolegroupref, vector_aggregator_address, - &config.logging, + &merged_config.logging, SparkHistoryServerContainer::SparkHistory, SparkHistoryServerContainer::Vector, &mut cm_builder, @@ -593,9 +624,10 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec { }); vars.push(EnvVar { name: "SPARK_HISTORY_OPTS".to_string(), - value: Some(format!( - "-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}" - )), + value: Some(vec![ + format!("-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"), + format!("-Djava.security.properties={VOLUME_MOUNT_PATH_LOG_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}"), + ].join(" ")), value_from: None, }); // if TLS is enabled build truststore