Support version 2.7.3, 2.8.1, remove 2.6.1, 2.7.1

CHANGELOG.md

@@ -6,13 +6,15 @@
 
 - More CRD documentation ([#354]).
 - Helm: support labels in values.yaml ([#374]).
+- Support for version `2.7.3`, `2.8.1` ([#387]).
 
 ### Changed
 
 - `operator-rs` `0.56.1` -> `0.57.0` ([#354]).
 - Increase resource defaults ([#352]).
 - Use new label builders ([#366]).
 - Use new ldap::AuthenticationClassProvider `endpoint_url()` method ([#366]).
+- Support git-sync `4.2.1` ([#387]).
 
 ### Fixed
 
@@ -23,6 +25,7 @@
 ### Removed
 
 - [BREAKING] Removed legacy node selector on roleGroups ([#366]).
+- Removed support for version `2.6.1`, `2.7.1` ([#387]).
 
 [#352]: https://github.com/stackabletech/airflow-operator/pull/352
 [#353]: https://github.com/stackabletech/airflow-operator/pull/353
@@ -31,6 +34,7 @@
 [#372]: https://github.com/stackabletech/airflow-operator/pull/372
 [#374]: https://github.com/stackabletech/airflow-operator/pull/374
 [#381]: https://github.com/stackabletech/airflow-operator/pull/381
+[#387]: https://github.com/stackabletech/airflow-operator/pull/387
 
 ## [23.11.0] - 2023-11-24
 

deploy/helm/airflow-operator/crds/crds.yaml

@@ -6955,7 +6955,7 @@ spec:
                       items:
                         properties:
                           branch:
-                            description: The branch to clone. Defaults to `main`.
+                            description: The branch to clone. Defaults to `main`. Since git-sync v4.x.x this field is mapped to the flag `--ref`.
                             nullable: true
                             type: string
                           credentialsSecret:
@@ -6982,7 +6982,7 @@ spec:
                             description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.'
                             type: string
                           wait:
-                            description: The synchronization interval in seconds; defaults to 20 seconds.
+                            description: The synchronization interval in seconds; defaults to 20 seconds. Since git-sync v4.x.x this field is mapped to the flag `--period`.
                             format: uint16
                             minimum: 0.0
                             nullable: true

docs/modules/airflow/examples/example-airflow-dags-configmap.yaml

@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
   clusterConfig:
     loadExamples: false
     exposeConfig: false

docs/modules/airflow/examples/example-airflow-gitsync.yaml

@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: "2.7.2"
+    productVersion: "2.8.1"
   clusterConfig:
     loadExamples: false
     exposeConfig: false

docs/modules/airflow/examples/example-airflow-incluster.yaml

@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
   clusterConfig:
     loadExamples: false
     exposeConfig: false

docs/modules/airflow/examples/getting_started/code/airflow.yaml

@@ -5,7 +5,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
   clusterConfig:
     loadExamples: true
     exposeConfig: false

docs/modules/airflow/pages/usage-guide/mounting-dags.adoc

@@ -27,7 +27,7 @@ The advantage of this approach is that a DAG can be provided "in-line", as it we
 
 === Overview
 
-https://github.com/kubernetes/git-sync/tree/release-3.x[git-sync] is a command that pulls a git repository into a local directory and is supplied as a sidecar container for use within Kubernetes. The Stackable implementation is a wrapper around this such that the binary and image requirements are included in the Stackable Airflow product images and do not need to be specified or handled in the `AirflowCluster` custom resource. Internal details such as image names and volume mounts are handled by the operator, so that only the repository and synchronisation details are required. An example of this usage is given in the next section.
+https://github.com/kubernetes/git-sync/tree/v4.2.1[git-sync] is a command that pulls a git repository into a local directory and is supplied as a sidecar container for use within Kubernetes. The Stackable implementation is a wrapper around this such that the binary and image requirements are included in the Stackable Airflow product images and do not need to be specified or handled in the `AirflowCluster` custom resource. Internal details such as image names and volume mounts are handled by the operator, so that only the repository and synchronization details are required. An example of this usage is given in the next section.
 
 === Example
 
@@ -44,7 +44,7 @@ include::example$example-airflow-gitsync.yaml[]
 <6> The depth of syncing i.e. the number of commits to clone (defaults to 1)
 <7> The synchronisation interval in seconds (defaults to 20 seconds)
 <8> The name of the `Secret` used to access the repository if it is not public. This should include two fields: `user` and `password` (which can be either a password - which is not recommended - or a github token, as described https://github.com/kubernetes/git-sync/tree/v3.6.4#flags-which-configure-authentication[here])
-<9> A map of optional configuration settings that are listed in https://github.com/kubernetes/git-sync/tree/v3.6.4#primary-flags[this] configuration section (and the ones that follow on that link)
+<9> A map of optional configuration settings that are listed in https://github.com/kubernetes/git-sync/tree/v4.2.1?tab=readme-ov-file#manual[this] configuration section (and the ones that follow on that link)
 <10> An example showing how to specify a target revision (the default is HEAD). The revision can also a be tag or a commit, though this assumes that the target hash is contained within the number of commits specified by `depth`. If a tag or commit hash is specified, then git-sync will recognise that and not perform further cloning.
 <11> Git-sync settings can be provided inline, although some of these (`--dest`, `--root`) are specified internally in the operator and will be ignored if provided by the user. Git-config settings can also be specified, although a warning will be logged if `safe.directory` is specified as this is defined internally, and should not be defined by the user.
 

docs/modules/airflow/pages/usage-guide/security.adoc

@@ -23,7 +23,7 @@ metadata:
   name: airflow-with-ldap
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
   clusterConfig:
     authentication:
       - authenticationClass: ldap    # <1>

docs/modules/airflow/partials/supported-versions.adoc

@@ -2,6 +2,7 @@
 // This is a separate file, since it is used by both the direct Airflow-Operator documentation, and the overarching
 // Stackable Platform documentation.
 
-- 2.7.2
-- 2.6.3
-- 2.6.1 (deprecated)
+- 2.8.1
+- 2.7.3 (deprecated)
+- 2.7.2 (deprecated)
+- 2.6.3 (LTS)

examples/simple-airflow-cluster-dags-cmap.yaml

@@ -81,7 +81,7 @@ metadata:
   name: airflow-dags-cmap
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: false

examples/simple-airflow-cluster-ldap-insecure-tls.yaml

@@ -147,7 +147,7 @@ metadata:
   name: airflow-insecure-tls
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: true

examples/simple-airflow-cluster-ldap.yaml

@@ -145,7 +145,7 @@ metadata:
   name: airflow-with-ldap-server-veri-tls
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: true

examples/simple-airflow-cluster.yaml

@@ -21,7 +21,7 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.7.2
+    productVersion: 2.8.1
     stackableVersion: 0.0.0-dev
   clusterConfig:
     loadExamples: true

rust/crd/src/affinity.rs

@@ -61,7 +61,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.7.2
+            productVersion: 2.8.1
           clusterConfig:
             credentialsSecret: airflow-credentials
           webservers:

rust/crd/src/git_sync.rs

@@ -5,20 +5,20 @@ use stackable_operator::{
 };
 use std::collections::BTreeMap;
 
-use crate::{GIT_LINK, GIT_ROOT, GIT_SAFE_DIR, GIT_SYNC_DEPTH, GIT_SYNC_WAIT};
+use crate::{GIT_LINK, GIT_ROOT, GIT_SAFE_DIR, GIT_SYNC_DEPTH, GIT_SYNC_PERIOD_SECONDS};
 
 #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct GitSync {
     /// The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.
     pub repo: String,
-    /// The branch to clone. Defaults to `main`.
+    /// The branch to clone. Defaults to `main`. Since git-sync v4.x.x this field is mapped to the flag `--ref`.
     pub branch: Option<String>,
     /// The location of the DAG folder, relative to the synced repository root.
     pub git_folder: Option<String>,
     /// The depth of syncing i.e. the number of commits to clone; defaults to 1.
     pub depth: Option<u8>,
-    /// The synchronization interval in seconds; defaults to 20 seconds.
+    /// The synchronization interval in seconds; defaults to 20 seconds. Since git-sync v4.x.x this field is mapped to the flag `--period`.
     pub wait: Option<u16>,
     /// The name of the Secret used to access the repository if it is not public.
     /// This should include two fields: `user` and `password`.
@@ -42,12 +42,12 @@ impl GitSync {
             "/stackable/git-sync".to_string(),
             format!("--repo={}", self.repo.clone()),
             format!(
-                "--branch={}",
+                "--ref={}",
                 self.branch.clone().unwrap_or_else(|| "main".to_string())
             ),
             format!("--depth={}", self.depth.unwrap_or(GIT_SYNC_DEPTH)),
-            format!("--wait={}", self.wait.unwrap_or(GIT_SYNC_WAIT)),
-            format!("--dest={GIT_LINK}"),
+            format!("--period={}s", self.wait.unwrap_or(GIT_SYNC_PERIOD_SECONDS)),
+            format!("--link={GIT_LINK}"),
             format!("--root={GIT_ROOT}"),
         ];
         if let Some(git_sync_conf) = self.git_sync_conf.as_ref() {
@@ -108,7 +108,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.7.2
+            productVersion: 2.8.1
           clusterConfig:
             loadExamples: false
             exposeConfig: false
@@ -154,7 +154,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.7.2
+            productVersion: 2.8.1
           clusterConfig:
             loadExamples: false
             exposeConfig: false
@@ -165,7 +165,7 @@ mod tests {
                 branch: feat/git-sync
                 wait: 20
                 gitSyncConf:
-                  --rev: c63921857618a8c392ad757dda13090fff3d879a
+                  --ref: c63921857618a8c392ad757dda13090fff3d879a
                 gitFolder: tests/templates/kuttl/mount-dags-gitsync/dags
           webservers:
             roleGroups:
@@ -190,7 +190,7 @@ mod tests {
             .unwrap()
             .get_args(false)
             .iter()
-            .any(|c| c.contains("--rev=c63921857618a8c392ad757dda13090fff3d879a")));
+            .any(|c| c.contains("--ref=c63921857618a8c392ad757dda13090fff3d879a")));
     }
 
     #[rstest]
@@ -227,7 +227,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.7.2
+            productVersion: 2.8.1
           clusterConfig:
             loadExamples: false
             exposeConfig: false

rust/crd/src/lib.rs

@@ -62,7 +62,7 @@ pub const TEMPLATE_LOCATION: &str = "/templates";
 pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml";
 
 const GIT_SYNC_DEPTH: u8 = 1u8;
-const GIT_SYNC_WAIT: u16 = 20u16;
+const GIT_SYNC_PERIOD_SECONDS: u16 = 20u16;
 
 const DEFAULT_AIRFLOW_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(2);
 const DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5);
@@ -772,7 +772,7 @@ mod tests {
           name: airflow
         spec:
           image:
-            productVersion: 2.7.2
+            productVersion: 2.8.1
           clusterConfig:
             loadExamples: true
             exposeConfig: true
@@ -796,7 +796,7 @@ mod tests {
         let resolved_airflow_image: ResolvedProductImage =
             cluster.spec.image.resolve("airflow", "0.0.0-dev");
 
-        assert_eq!("2.7.2", &resolved_airflow_image.product_version);
+        assert_eq!("2.8.1", &resolved_airflow_image.product_version);
 
         assert_eq!("KubernetesExecutor", cluster.spec.executor.to_string());
         assert!(cluster.spec.cluster_config.load_examples);

rust/operator-binary/src/airflow_controller.rs

@@ -1105,12 +1105,12 @@ fn build_executor_template_config_map(
         let mut env = vec![];
         if let Some(credentials_secret) = &gitsync.credentials_secret {
             env.push(env_var_from_secret(
-                "GIT_SYNC_USERNAME",
+                "GITSYNC_USERNAME",
                 credentials_secret,
                 "user",
             ));
             env.push(env_var_from_secret(
-                "GIT_SYNC_PASSWORD",
+                "GITSYNC_PASSWORD",
                 credentials_secret,
                 "password",
             ));
@@ -1276,23 +1276,11 @@ fn build_mapped_envs(
             value: Some(format!("{TEMPLATE_LOCATION}/{TEMPLATE_NAME}")),
             ..Default::default()
         });
-        // TODO: version < 2.5, delete when these versions are no longer supported
-        env.push(EnvVar {
-            name: "AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE".into(),
-            value: Some(format!("{TEMPLATE_LOCATION}/{TEMPLATE_NAME}")),
-            ..Default::default()
-        });
         env.push(EnvVar {
             name: "AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE".into(),
             value: airflow.namespace(),
             ..Default::default()
         });
-        // TODO: version < 2.5, delete when these versions are no longer supported
-        env.push(EnvVar {
-            name: "AIRFLOW__KUBERNETES__NAMESPACE".into(),
-            value: airflow.namespace(),
-            ..Default::default()
-        });
     }
 
     env
@@ -1324,12 +1312,6 @@ fn build_template_envs(
         value: airflow.namespace(),
         ..Default::default()
     });
-    // TODO: version < 2.5, delete when these versions are no longer supported
-    env.push(EnvVar {
-        name: "AIRFLOW__KUBERNETES__NAMESPACE".into(),
-        value: airflow.namespace(),
-        ..Default::default()
-    });
 
     // iterate over a BTreeMap to ensure the vars are written in a predictable order
     for (k, v) in env_overrides.iter().collect::<BTreeMap<_, _>>() {
@@ -1361,9 +1343,9 @@ fn build_gitsync_envs(
         .get(&PropertyNameKind::Env)
         .and_then(|vars| vars.get(AirflowConfig::GIT_CREDENTIALS_SECRET_PROPERTY))
     {
-        env.push(env_var_from_secret("GIT_SYNC_USERNAME", git_secret, "user"));
+        env.push(env_var_from_secret("GITSYNC_USERNAME", git_secret, "user"));
         env.push(env_var_from_secret(
-            "GIT_SYNC_PASSWORD",
+            "GITSYNC_PASSWORD",
             git_secret,
             "password",
         ));

tests/test-definition.yaml

@@ -7,12 +7,12 @@
 dimensions:
   - name: airflow
     values:
-      - 2.6.1
       - 2.6.3
-      - 2.7.2
+      - 2.7.3
+      - 2.8.1
   - name: airflow-latest
     values:
-      - 2.7.2
+      - 2.8.1
   - name: ldap-authentication
     values:
       - no-tls