(teamclairvoyant#133) feat: add doc_md

README.md

@@ -3,18 +3,18 @@ A series of DAGs/Workflows to help maintain the operation of Airflow
 
 ## DAGs/Workflows
 
-* [backup-configs](backup-configs)
+* [backup-configs](airflow-backup-configs)
     * A maintenance workflow that you can deploy into Airflow to periodically take backups of various Airflow configurations and files.
-* [clear-missing-dags](clear-missing-dags)
+* [clear-missing-dags](airflow-clear-missing-dags)
     * A maintenance workflow that you can deploy into Airflow to periodically clean out entries in the DAG table of which there is no longer a corresponding Python File for it. This ensures that the DAG table doesn't have needless items in it and that the Airflow Web Server displays only those available DAGs.
-* [db-cleanup](db-cleanup)
+* [db-cleanup](airflow-db-cleanup)
     * A maintenance workflow that you can deploy into Airflow to periodically clean out the DagRun, TaskInstance, Log, XCom, Job DB and SlaMiss entries to avoid having too much data in your Airflow MetaStore.
-* [kill-halted-tasks](kill-halted-tasks)
+* [kill-halted-tasks](airflow-kill-halted-tasks)
     * A maintenance workflow that you can deploy into Airflow to periodically kill off tasks that are running in the background that don't correspond to a running task in the DB.
     * This is useful because when you kill off a DAG Run or Task through the Airflow Web Server, the task still runs in the background on one of the executors until the task is complete.
-* [log-cleanup](log-cleanup)
+* [log-cleanup](airflow-log-cleanup)
     * A maintenance workflow that you can deploy into Airflow to periodically clean out the task logs to avoid those getting too big.
-* [delete-broken-dags](delete-broken-dags)
+* [delete-broken-dags](airflow-delete-broken-dags)
     * A maintenance workflow that you can deploy into Airflow to periodically delete DAG files and clean out entries in the ImportError table for DAGs which Airflow cannot parse or import properly. This ensures that the ImportError table is cleaned every day.
-* [sla-miss-report](sla-miss-report)
+* [sla-miss-report](airflow-sla-miss-report)
     * DAG providing an extensive analysis report of SLA misses broken down on a daily, hourly, and task level

backup-configs/airflow-backup-configs.py → airflow-backup-configs/airflow-backup-configs.py

@@ -15,6 +15,10 @@
 import subprocess
 # airflow-backup-configs
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 # How often to Run. @daily - Once a day at Midnight
 START_DATE = airflow.utils.dates.days_ago(1)
 # Who is listed as the owner of this DAG in the Airflow Web Server
@@ -52,7 +56,7 @@
     tags=['teamclairvoyant', 'airflow-maintenance-dags']
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

clear-missing-dags/airflow-clear-missing-dags.py → airflow-clear-missing-dags/airflow-clear-missing-dags.py

@@ -17,6 +17,10 @@
 
 # airflow-clear-missing-dags
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 # How often to Run. @daily - Once a day at Midnight
 SCHEDULE_INTERVAL = "@daily"
@@ -47,7 +51,7 @@
     tags=['teamclairvoyant', 'airflow-maintenance-dags']
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

db-cleanup/airflow-db-cleanup.py → airflow-db-cleanup/airflow-db-cleanup.py

@@ -35,6 +35,10 @@
 
 # airflow-db-cleanup
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 # How often to Run. @daily - Once a day at Midnight (UTC)
 SCHEDULE_INTERVAL = "@daily"
@@ -216,7 +220,7 @@
     tags=['teamclairvoyant', 'airflow-maintenance-dags']
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

delete-broken-dags/airflow-delete-broken-dags.py → airflow-delete-broken-dags/airflow-delete-broken-dags.py

@@ -17,6 +17,10 @@
 
 # airflow-delete-broken-dags
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 # How often to Run. @daily - Once a day at Midnight
 SCHEDULE_INTERVAL = "@daily"
@@ -46,7 +50,7 @@
     tags=['teamclairvoyant', 'airflow-maintenance-dags']
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

kill-halted-tasks/airflow-kill-halted-tasks.py → airflow-kill-halted-tasks/airflow-kill-halted-tasks.py

@@ -26,6 +26,10 @@
 
 # airflow-kill-halted-tasks
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 # How often to Run. @daily - Once a day at Midnight. @hourly - Once an Hour.
 SCHEDULE_INTERVAL = "@hourly"
@@ -64,7 +68,7 @@
     tags=['teamclairvoyant', 'airflow-maintenance-dags']
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

log-cleanup/airflow-log-cleanup-pwdless-ssh.py → airflow-log-cleanup/airflow-log-cleanup-pwdless-ssh.py

@@ -18,6 +18,10 @@
 
 # airflow-log-cleanup
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 try:
     BASE_LOG_FOLDER = conf.get("core", "BASE_LOG_FOLDER").rstrip("/")
@@ -87,7 +91,7 @@
     tags=['teamclairvoyant', 'airflow-maintenance-dags']
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

log-cleanup/airflow-log-cleanup.py → airflow-log-cleanup/airflow-log-cleanup.py

@@ -18,6 +18,10 @@
 
 # airflow-log-cleanup
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 try:
     BASE_LOG_FOLDER = conf.get("core", "BASE_LOG_FOLDER").rstrip("/")
@@ -88,7 +92,7 @@
     template_undefined=jinja2.Undefined
 )
 if hasattr(dag, 'doc_md'):
-    dag.doc_md = __doc__
+    dag.doc_md = DOC_MD
 if hasattr(dag, 'catchup'):
     dag.catchup = False
 

sla-miss-report/airflow-sla-miss-report.py → airflow-sla-miss-report/airflow-sla-miss-report.py

@@ -16,6 +16,10 @@
 ################################
 
 DAG_ID = os.path.basename(__file__).replace(".pyc", "").replace(".py", "")
+DOC_MD = f"""
+### [README.md](https://github.com/teamclairvoyant/airflow-maintenance-dags/tree/master/{DAG_ID})
+"""
+
 START_DATE = airflow.utils.dates.days_ago(1)
 # How often to Run. @daily - Once a day at Midnight
 SCHEDULE_INTERVAL = "@daily"
@@ -751,5 +755,6 @@ def no_metadata_found():
          description="DAG generating the SLA miss report",
          schedule_interval=SCHEDULE_INTERVAL,
          start_date=START_DATE,
-         tags=['teamclairvoyant', 'airflow-maintenance-dags']) as dag:
+         tags=['teamclairvoyant', 'airflow-maintenance-dags'],
+         doc_md=DOC_MD) as dag:
     sla_miss_report_task = PythonOperator(task_id="sla_miss_report", python_callable=sla_miss_report, dag=dag)