diff --git a/docs/modules/airflow/pages/usage-guide/logging.adoc b/docs/modules/airflow/pages/usage-guide/logging.adoc index c1a1150c..9641dd5e 100644 --- a/docs/modules/airflow/pages/usage-guide/logging.adoc +++ b/docs/modules/airflow/pages/usage-guide/logging.adoc @@ -17,7 +17,7 @@ spec: loggers: "flask_appbuilder": level: WARN - workers: + celeryExecutors: config: logging: enableVectorAgent: true diff --git a/docs/modules/airflow/pages/usage-guide/storage-resources.adoc b/docs/modules/airflow/pages/usage-guide/storage-resources.adoc index 39aa2d77..1a2a61b4 100644 --- a/docs/modules/airflow/pages/usage-guide/storage-resources.adoc +++ b/docs/modules/airflow/pages/usage-guide/storage-resources.adoc @@ -21,7 +21,7 @@ spec: max: "2" memory: limit: 512Mi - workers: + celeryExecutors: config: resources: cpu: diff --git a/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc b/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc new file mode 100644 index 00000000..a3717991 --- /dev/null +++ b/docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc @@ -0,0 +1,29 @@ += Using Kubernetes executors + +Instead of using the Celery workers you can let Airflow run the tasks using Kubernetes executors, where pods are created dynamically as needed without jobs being routed through a redis queue to the workers. + +To achieve this, swap `spec.celeryExecutors` with `spec.kubernetesExecutors`. +E.g. you would change the following example + +[source,yaml] +---- +spec: + celeryExecutors: + roleGroups: + default: + replicas: 2 + config: + resources: + # ... +---- + +to + +[source,yaml] +---- +spec: + kubernetesExecutors: + config: + resources: + # ... +---- diff --git a/docs/modules/airflow/partials/nav.adoc b/docs/modules/airflow/partials/nav.adoc index 4ba6a6bb..c4a54d5e 100644 --- a/docs/modules/airflow/partials/nav.adoc +++ b/docs/modules/airflow/partials/nav.adoc @@ -10,6 +10,7 @@ ** xref:airflow:usage-guide/security.adoc[] ** xref:airflow:usage-guide/logging.adoc[] ** xref:airflow:usage-guide/monitoring.adoc[] +** xref:airflow:usage-guide/using-kubernetes-executors.adoc[] ** xref:airflow:usage-guide/overrides.adoc[] ** xref:airflow:usage-guide/operations/index.adoc[] *** xref:airflow:usage-guide/operations/cluster-operations.adoc[]