apache#45037: Support for additional celery config directly from airflow.cfg file

Author: sachin-arora-cashfree

chart/values.yaml

@@ -2607,6 +2607,7 @@ config:
   celery:
     flower_url_prefix: '{{ ternary "" .Values.ingress.flower.path (eq .Values.ingress.flower.path "/") }}'
     worker_concurrency: 16
+    extra_celery_config: '{}'
   scheduler:
     standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}'
     # statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0

providers/src/airflow/providers/celery/executors/default_celery.py

@@ -69,6 +69,8 @@ def _broker_supports_visibility_timeout(url):
     log.debug("Value for celery result_backend not found. Using sql_alchemy_conn with db+ prefix.")
     result_backend = f'db+{conf.get("database", "SQL_ALCHEMY_CONN")}'
 
+extra_celery_config: dict = conf.getjson("celery", "extra_celery_config", fallback={}) if conf.has_option("celery", "extra_celery_config") else {}
+
 DEFAULT_CELERY_CONFIG = {
     "accept_content": ["json"],
     "event_serializer": "json",
@@ -85,6 +87,7 @@ def _broker_supports_visibility_timeout(url):
     ),
     "worker_concurrency": conf.getint("celery", "WORKER_CONCURRENCY", fallback=16),
     "worker_enable_remote_control": conf.getboolean("celery", "worker_enable_remote_control", fallback=True),
+    **extra_celery_config
 }