feat: Support graceful shutdown

CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 - Default stackableVersion to operator version ([#360]).
 - Configuration overrides for the JVM security properties, such as DNS caching ([#365]).
 - Support PodDisruptionBudgets ([#376]).
+- Support graceful shutdown ([#385]).
 
 ### Changed
 
@@ -27,6 +28,7 @@ All notable changes to this project will be documented in this file.
 [#376]: https://github.com/stackabletech/hive-operator/pull/376
 [#377]: https://github.com/stackabletech/hive-operator/pull/377
 [#382]: https://github.com/stackabletech/hive-operator/pull/382
+[#385]: https://github.com/stackabletech/hive-operator/pull/385
 
 ## [23.7.0] - 2023-07-14
 

deploy/helm/hive-operator/crds/crds.yaml

@@ -724,6 +724,10 @@ spec:
                                   type: array
                               type: object
                           type: object
+                        gracefulShutdownTimeout:
+                          description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -4213,6 +4217,10 @@ spec:
                                         type: array
                                     type: object
                                 type: object
+                              gracefulShutdownTimeout:
+                                description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null

docs/modules/hive/pages/usage-guide/operations/graceful-shutdown.adoc

@@ -1,7 +1,12 @@
 = Graceful shutdown
 
-Graceful shutdown of Zookeeper nodes is either not supported by the product itself
-or we have not implemented it yet.
+You can configure the graceful shutdown as described in xref:concepts:operations/graceful_shutdown.adoc[].
 
-Outstanding implementation work for the graceful shutdowns of all products where this functionality is relevant is tracked in
-https://github.com/stackabletech/issues/issues/357
+== Hive metastores
+
+As a default, Hive metastores have `5 minutes` to shut down gracefully.
+
+The Hive metastores process will receive a `SIGTERM` signal when Kubernetes wants to terminate the Pod.
+After the graceful shutdown timeout runs out, and the process still didn't exit, Kubernetes will issue a `SIGKILL` signal.
+
+However, there is no acknowledge message in the log indicating a graceful shutdown.

rust/crd/src/lib.rs

@@ -18,10 +18,16 @@ use stackable_operator::{
     k8s_openapi::apimachinery::pkg::api::resource::Quantity,
     kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt},
     product_config_utils::{ConfigError, Configuration},
-    product_logging::{self, spec::Logging},
+    product_logging::{
+        self,
+        framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command},
+        spec::Logging,
+    },
     role_utils::{GenericRoleConfig, Role, RoleGroup, RoleGroupRef},
     schemars::{self, JsonSchema},
     status::condition::{ClusterCondition, HasStatusCondition},
+    time::Duration,
+    utils::COMMON_BASH_TRAP_FUNCTIONS,
 };
 use strum::{Display, EnumIter, EnumString, IntoEnumIterator};
 
@@ -62,6 +68,8 @@ pub const HIVE_METASTORE_HADOOP_OPTS: &str = "HIVE_METASTORE_HADOOP_OPTS";
 pub const HADOOP_HEAPSIZE: &str = "HADOOP_HEAPSIZE";
 pub const JVM_HEAP_FACTOR: f32 = 0.8;
 
+const DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5);
+
 #[derive(Snafu, Debug)]
 pub enum Error {
     #[snafu(display("no metastore role configuration provided"))]
@@ -181,25 +189,19 @@ pub enum HiveRole {
 
 impl HiveRole {
     /// Returns the container start command for the metastore service.
-    pub fn get_command(&self, auto_init_schema: bool, db_type: &str) -> Vec<String> {
-        if auto_init_schema {
-            vec![
-                "bin/start-metastore".to_string(),
-                "--config".to_string(),
-                STACKABLE_CONFIG_DIR.to_string(),
-                "--db-type".to_string(),
-                db_type.to_string(),
-                "--hive-bin-dir".to_string(),
-                "bin".to_string(),
-            ]
-        } else {
-            vec![
-                "/bin/hive".to_string(),
-                "--config".to_string(),
-                STACKABLE_CONFIG_DIR.to_string(),
-                "--service".to_string(),
-                "metastore".to_string(),
-            ]
+    pub fn get_command(&self, db_type: &str) -> String {
+        formatdoc! {"
+            {COMMON_BASH_TRAP_FUNCTIONS}
+            {remove_vector_shutdown_file_command}
+            prepare_signal_handlers
+            bin/start-metastore --config {STACKABLE_CONFIG_DIR} --db-type {db_type} --hive-bin-dir bin &
+            wait_for_termination $!
+            {create_vector_shutdown_file_command}
+            ",
+            remove_vector_shutdown_file_command =
+                remove_vector_shutdown_file_command(STACKABLE_LOG_DIR),
+            create_vector_shutdown_file_command =
+                create_vector_shutdown_file_command(STACKABLE_LOG_DIR),
         }
     }
 
@@ -284,12 +286,19 @@ pub struct MetaStoreConfig {
     /// The location of default database for the Hive warehouse.
     /// Maps to the `hive.metastore.warehouse.dir` setting.
     pub warehouse_dir: Option<String>,
+
     #[fragment_attrs(serde(default))]
     pub resources: Resources<MetastoreStorageConfig, NoRuntimeLimits>,
+
     #[fragment_attrs(serde(default))]
     pub logging: Logging<Container>,
+
     #[fragment_attrs(serde(default))]
     pub affinity: StackableAffinity,
+
+    /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+    #[fragment_attrs(serde(default))]
+    pub graceful_shutdown_timeout: Option<Duration>,
 }
 
 impl MetaStoreConfig {
@@ -330,6 +339,7 @@ impl MetaStoreConfig {
             },
             logging: product_logging::spec::default_logging(),
             affinity: get_affinity(cluster_name, role),
+            graceful_shutdown_timeout: Some(DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT),
         }
     }
 }

rust/operator-binary/src/command.rs

@@ -50,8 +50,9 @@ pub fn build_container_command_args(
             args.push(format!("keytool -importcert -file /stackable/certificates/{secret_class}-tls-certificate/ca.crt -alias stackable-{secret_class} -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt"));
         }
     }
+
     // metastore start command
     args.push(start_command);
 
-    vec![args.join(" && ")]
+    vec![args.join("\n")]
 }

rust/operator-binary/src/controller.rs

@@ -73,7 +73,7 @@ use tracing::warn;
 use crate::{
     command::{self, build_container_command_args, S3_SECRET_DIR},
     discovery,
-    operations::pdb::add_pdbs,
+    operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs},
     product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address},
     OPERATOR_NAME,
 };
@@ -254,6 +254,11 @@ pub enum Error {
     FailedToCreatePdb {
         source: crate::operations::pdb::Error,
     },
+
+    #[snafu(display("failed to configure graceful shutdown"))]
+    GracefulShutdown {
+        source: crate::operations::graceful_shutdown::Error,
+    },
 }
 type Result<T, E = Error> = std::result::Result<T, E>;
 
@@ -805,14 +810,13 @@ fn build_metastore_rolegroup_statefulset(
         .image_from_product_image(resolved_product_image)
         .command(vec![
             "/bin/bash".to_string(),
-            "-c".to_string(),
+            "-x".to_string(),
             "-euo".to_string(),
             "pipefail".to_string(),
+            "-c".to_string(),
         ])
         .args(build_container_command_args(
-            HiveRole::MetaStore
-                .get_command(true, &db_type.unwrap_or_default().to_string())
-                .join(" "),
+            HiveRole::MetaStore.get_command(&db_type.unwrap_or_default().to_string()),
             s3_connection,
         ))
         .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR)
@@ -940,6 +944,8 @@ fn build_metastore_rolegroup_statefulset(
         ));
     }
 
+    add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?;
+
     let mut pod_template = pod_builder.build_template();
     pod_template.merge_from(role.config.pod_overrides.clone());
     pod_template.merge_from(rolegroup.config.pod_overrides.clone());

rust/operator-binary/src/operations/graceful_shutdown.rs

@@ -0,0 +1,26 @@
+use snafu::{ResultExt, Snafu};
+use stackable_hive_crd::MetaStoreConfig;
+use stackable_operator::builder::PodBuilder;
+
+#[derive(Debug, Snafu)]
+pub enum Error {
+    #[snafu(display("Failed to set terminationGracePeriod"))]
+    SetTerminationGracePeriod {
+        source: stackable_operator::builder::pod::Error,
+    },
+}
+
+pub fn add_graceful_shutdown_config(
+    merged_config: &MetaStoreConfig,
+    pod_builder: &mut PodBuilder,
+) -> Result<(), Error> {
+    // This must be always set by the merge mechanism, as we provide a default value,
+    // users can not disable graceful shutdown.
+    if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout {
+        pod_builder
+            .termination_grace_period(&graceful_shutdown_timeout)
+            .context(SetTerminationGracePeriodSnafu)?;
+    }
+
+    Ok(())
+}

rust/operator-binary/src/operations/mod.rs

@@ -1 +1,2 @@
+pub mod graceful_shutdown;
 pub mod pdb;

tests/templates/kuttl/smoke/60-assert.yaml.j2

@@ -22,6 +22,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
         - name: vector
 {% endif %}
+      terminationGracePeriodSeconds: 300
 status:
   readyReplicas: 1
   replicas: 1