Better CRD docs

CHANGELOG.md

@@ -2,16 +2,22 @@
 
 ## [Unreleased]
 
+### Added
+
+- More CRD documentation ([#354]).
+
 ### Changed
 
+- `operator-rs` `0.56.1` -> `0.57.0` ([#354]).
 - Increase resource defaults ([#352]).
 
 ### Fixed
 
 - BREAKING: Fixed various issues in the CRD structure. `clusterConfig.credentialsSecret` is now mandatory ([#353]).
 
-[#352]: https://github.com/stackabletech/airflow-operator/pull/352
+[#354]: https://github.com/stackabletech/airflow-operator/pull/354
 [#353]: https://github.com/stackabletech/airflow-operator/pull/353
+[#352]: https://github.com/stackabletech/airflow-operator/pull/352
 
 ## [23.11.0] - 2023-11-24
 

Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.56.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.57.0" }
 strum = { version = "0.25", features = ["derive"] }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"

rust/crd/src/authentication.rs

@@ -42,18 +42,18 @@ pub struct AirflowAuthenticationConfigResolved {
 #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct AirflowAuthentication {
-    /// The Airflow authentication settings.
+    /// The Airflow [authentication](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/security.html) settings.
     /// Currently the underlying Flask App Builder only supports one authentication mechanism
     /// at a time. This means the operator will error out if multiple references to an
-    /// `AuthenticationClass` are provided.
+    /// AuthenticationClass are provided.
     #[serde(default)]
     authentication: Vec<AirflowAuthenticationConfig>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct AirflowAuthenticationConfig {
-    /// Name of the AuthenticationClass used to authenticate the users.
+    /// Name of the [AuthenticationClass](DOCS_BASE_URL_PLACEHOLDER/concepts/authentication.html#authenticationclass) used to authenticate the users.
     /// At the moment only LDAP is supported.
     /// If not specified the default authentication (AUTH_DB) will be used.
     pub authentication_class: Option<String>,

rust/crd/src/git_sync.rs

@@ -10,12 +10,23 @@ use crate::{GIT_LINK, GIT_ROOT, GIT_SAFE_DIR, GIT_SYNC_DEPTH, GIT_SYNC_WAIT};
 #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct GitSync {
+    /// The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.
     pub repo: String,
+    /// The branch to clone. Defaults to `main`.
     pub branch: Option<String>,
+    /// The location of the DAG folder, relative to the synced repository root.
     pub git_folder: Option<String>,
+    /// The depth of syncing i.e. the number of commits to clone; defaults to 1.
     pub depth: Option<u8>,
+    /// The synchronization interval in seconds; defaults to 20 seconds.
     pub wait: Option<u16>,
+    /// The name of the Secret used to access the repository if it is not public.
+    /// This should include two fields: `user` and `password`.
+    /// The `password` field can either be an actual password (not recommended) or a GitHub token,
+    /// as described [here](https://github.com/kubernetes/git-sync/tree/v3.6.4#flags-which-configure-authentication).
     pub credentials_secret: Option<String>,
+    /// A map of optional configuration settings that are listed in the [git-sync documentation](https://github.com/kubernetes/git-sync/tree/v3.6.4#primary-flags).
+    /// Read the [git sync example](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_example).
     pub git_sync_conf: Option<BTreeMap<String, String>>,
 }
 impl GitSync {

rust/crd/src/lib.rs

@@ -132,6 +132,12 @@ impl FlaskAppConfigOptions for AirflowConfigOptions {
     }
 }
 
+/// An Airflow cluster stacklet. This resource is managed by the Stackable operator for Apache Airflow.
+/// Find more information on how to use it and the resources that the operator generates in the
+/// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/).
+///
+/// The CRD contains three roles: webserver, scheduler and worker/celeryExecutor.
+/// Alternatively to the celeryExecutor you can use the kubernetesExecutor.
 #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
 #[kube(
     group = "airflow.stackable.tech",
@@ -149,19 +155,23 @@ impl FlaskAppConfigOptions for AirflowConfigOptions {
 )]
 #[serde(rename_all = "camelCase")]
 pub struct AirflowClusterSpec {
-    /// The Airflow image to use
+    // no doc string - See ProductImage struct
     pub image: ProductImage,
 
-    /// Global cluster configuration that applies to all roles and role groups
+    /// Configuration that applies to all roles and role groups.
+    /// This includes settings for authentication, git sync, service exposition and volumes, among other things.
     pub cluster_config: AirflowClusterConfig,
 
-    /// Cluster operations like pause reconciliation or cluster stop.
+    // no doc string - See ClusterOperation struct
     #[serde(default)]
     pub cluster_operation: ClusterOperation,
 
+    /// The `webserver` role provides the main UI for user interaction.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub webservers: Option<Role<AirflowConfigFragment>>,
 
+    /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database.
+    /// Jobs are scheduled on the workers/executors.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub schedulers: Option<Role<AirflowConfigFragment>>,
 
@@ -174,18 +184,23 @@ pub struct AirflowClusterSpec {
 pub struct AirflowClusterConfig {
     #[serde(flatten)]
     pub authentication: AirflowAuthentication,
-
+    /// The name of the Secret object containing the admin user credentials and database connection details.
+    /// Read the
+    /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/first_steps)
+    /// to find out more.
     pub credentials_secret: String,
-
+    /// The `gitSync` settings allow configuring DAGs to mount via `git-sync`.
+    /// Learn more in the
+    /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync).
     #[serde(default)]
     pub dags_git_sync: Vec<GitSync>,
-
+    /// for internal use only - not for production use.
     #[serde(default)]
     pub expose_config: bool,
-
+    /// Wether to load example DAGs or not; defaults to false. The examples are used in the
+    /// [getting started guide](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/).
     #[serde(default)]
     pub load_examples: bool,
-
     /// This field controls which type of Service the Operator creates for this AirflowCluster:
     ///
     /// * cluster-internal: Use a ClusterIP service
@@ -195,19 +210,23 @@ pub struct AirflowClusterConfig {
     /// * external-stable: Use a LoadBalancer service
     ///
     /// This is a temporary solution with the goal to keep yaml manifests forward compatible.
-    /// In the future, this setting will control which ListenerClass <https://docs.stackable.tech/home/stable/listener-operator/listenerclass.html>
+    /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html)
     /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change.
     #[serde(default)]
     pub listener_class: CurrentlySupportedListenerClasses,
 
-    /// Name of the Vector aggregator discovery ConfigMap.
+    /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery).
     /// It must contain the key `ADDRESS` with the address of the Vector aggregator.
+    /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator)
+    /// to learn how to configure log aggregation with Vector.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub vector_aggregator_config_map_name: Option<String>,
 
+    /// Additional volumes to define. Use together with `volumeMounts` to mount the volumes.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volumes: Option<Vec<Volume>>,
 
+    /// Additional volumes to mount. Use together with `volumes` to define volumes.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volume_mounts: Option<Vec<VolumeMount>>,
 }
@@ -352,12 +371,15 @@ impl AirflowRole {
 
 #[derive(Clone, Debug, Deserialize, Display, JsonSchema, PartialEq, Serialize)]
 pub enum AirflowExecutor {
+    /// The celery executor.
+    /// Deployed with an explicit number of replicas.
     #[serde(rename = "celeryExecutors")]
     CeleryExecutor {
         #[serde(flatten)]
         config: Role<AirflowConfigFragment>,
     },
 
+    /// With the Kuberentes executor, executor Pods are created on demand.
     #[serde(rename = "kubernetesExecutors")]
     KubernetesExecutor {
         #[serde(flatten)]

rust/operator-binary/src/main.rs

@@ -44,7 +44,7 @@ async fn main() -> anyhow::Result<()> {
 
     match opts.cmd {
         Command::Crd => {
-            AirflowCluster::print_yaml_schema()?;
+            AirflowCluster::print_yaml_schema(built_info::CARGO_PKG_VERSION)?;
         }
         Command::Run(ProductOperatorRun {
             product_config,