From db14460b54f85ab8ac3d555d176ef46b20f08268 Mon Sep 17 00:00:00 2001
From: Harshith-umesh <harshith.umesh.nat@gmail.com>
Date: Wed, 22 Jun 2022 13:15:43 +0530
Subject: [PATCH 1/4] pass rosa install vars

---
 dags/openshift_nightlies/scripts/install/rosa.sh | 13 +++++++++++++
 .../openshift_nightlies/scripts/run_benchmark.sh | 16 ++++++++++++++++
 dags/openshift_nightlies/tasks/benchmarks/e2e.py |  2 +-
 .../tasks/install/rosa/rosa.py                   |  2 +-
 .../tasks/utils/final_dag_status.py              |  1 +
 5 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/dags/openshift_nightlies/scripts/install/rosa.sh b/dags/openshift_nightlies/scripts/install/rosa.sh
index f3ee70f0d..086156c0a 100755
--- a/dags/openshift_nightlies/scripts/install/rosa.sh
+++ b/dags/openshift_nightlies/scripts/install/rosa.sh
@@ -338,6 +338,17 @@ EOF
     return 0
 }
 
+display_install_data(){
+    IFS='@'
+    # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
+    read -ra newarr <<< "$InstallUUID"
+
+    clustername="${newarr[0]}"
+    installuuid="${newarr[1]}"
+    echo "Cluster Name = ${clustername}"
+    echo "Install UUID = ${installuuid}"
+}
+
 cleanup(){
     if [[ $INSTALL_METHOD == "osd" ]]; then
         ocm delete cluster $(_get_cluster_id ${CLUSTER_NAME})
@@ -377,9 +388,11 @@ if [[ "$operation" == "install" ]]; then
         printf "INFO: Cluster ${CLUSTER_NAME} already installed but not ready, exiting..."
 	    exit 1
     fi
+    echo "${CLUSTER_NAME}@${UUID}"
 
 elif [[ "$operation" == "cleanup" ]]; then
     printf "Running Cleanup Steps"
+    display_install_data
     cleanup
     index_metadata
     rosa logout
diff --git a/dags/openshift_nightlies/scripts/run_benchmark.sh b/dags/openshift_nightlies/scripts/run_benchmark.sh
index 1a17f7158..9a444e3df 100755
--- a/dags/openshift_nightlies/scripts/run_benchmark.sh
+++ b/dags/openshift_nightlies/scripts/run_benchmark.sh
@@ -70,6 +70,21 @@ EOF
         exit 1
     fi
 }
+
+display_install_data(){    
+    if [[ $PLATFORM == "rosa" ]]; then
+        IFS='@'
+        # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
+        read -ra newarr <<< "$InstallUUID"
+
+        clustername="${newarr[0]}"
+        installuuid="${newarr[1]}"
+        echo "Cluster Name = ${clustername}"
+        echo "Install UUID = ${installuuid}"
+    fi
+}
+
+
 export UUID=$(uuidgen | head -c8)-$AIRFLOW_CTX_TASK_ID-$(date '+%Y%m%d')
 echo "############################################"
 echo "# Benchmark UUID: ${UUID}"
@@ -80,6 +95,7 @@ if [[ $PLATFORM == "baremetal" ]]; then
     run_baremetal_benchmark
     echo $UUID
 else
+    display_install_data
     setup
     cd /home/airflow/workspace/e2e-benchmarking/workloads/$workload
     
diff --git a/dags/openshift_nightlies/tasks/benchmarks/e2e.py b/dags/openshift_nightlies/tasks/benchmarks/e2e.py
index 9bfbbd46f..b6429430a 100644
--- a/dags/openshift_nightlies/tasks/benchmarks/e2e.py
+++ b/dags/openshift_nightlies/tasks/benchmarks/e2e.py
@@ -128,7 +128,7 @@ def _get_benchmark(self, benchmark):
                 retries=0,
                 trigger_rule=benchmark.get("trigger_rule", "all_success"),
                 dag=self.dag,
-                env=env,
+                env={ **env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'},
                 do_xcom_push=True,
                 execution_timeout=timedelta(seconds=21600),
                 executor_config=self.exec_config
diff --git a/dags/openshift_nightlies/tasks/install/rosa/rosa.py b/dags/openshift_nightlies/tasks/install/rosa/rosa.py
index cb1e0ea1b..f8668a88e 100644
--- a/dags/openshift_nightlies/tasks/install/rosa/rosa.py
+++ b/dags/openshift_nightlies/tasks/install/rosa/rosa.py
@@ -35,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"):
             dag=self.dag,
             trigger_rule=trigger_rule,
             executor_config=self.exec_config,
-            env=self.env
+            env={ **self.env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'}
         )
diff --git a/dags/openshift_nightlies/tasks/utils/final_dag_status.py b/dags/openshift_nightlies/tasks/utils/final_dag_status.py
index 41b759239..19c90b39a 100644
--- a/dags/openshift_nightlies/tasks/utils/final_dag_status.py
+++ b/dags/openshift_nightlies/tasks/utils/final_dag_status.py
@@ -8,6 +8,7 @@
 
 
 def final_status(**kwargs):
+    ti = kwargs['ti']
     failed_tasks=[]
     for task_instance in kwargs['dag_run'].get_task_instances():
         if "index" in task_instance.task_id:

From fcc3d60f809204bab9da38cc0c3508b501cbb032 Mon Sep 17 00:00:00 2001
From: Harshith-umesh <harshith.umesh.nat@gmail.com>
Date: Mon, 27 Jun 2022 14:56:19 +0530
Subject: [PATCH 2/4] adding xcom to rogcp

---
 dags/openshift_nightlies/scripts/install/ocm_gcp.sh   | 1 +
 dags/openshift_nightlies/scripts/run_benchmark.sh     | 2 +-
 dags/openshift_nightlies/tasks/benchmarks/e2e.py      | 4 ++--
 dags/openshift_nightlies/tasks/install/rogcp/rogcp.py | 3 ++-
 dags/openshift_nightlies/tasks/install/rosa/rosa.py   | 4 ++--
 5 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
index 630a7b4af..960686f4b 100755
--- a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
+++ b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
@@ -144,6 +144,7 @@ if [[ "$operation" == "install" ]]; then
         printf "INFO: Cluster ${CLUSTER_NAME} already installed but not ready, exiting..."
 	exit 1
     fi
+    echo "${CLUSTER_NAME}@${UUID}"
 
 elif [[ "$operation" == "cleanup" ]]; then
     printf "Running Cleanup Steps"
diff --git a/dags/openshift_nightlies/scripts/run_benchmark.sh b/dags/openshift_nightlies/scripts/run_benchmark.sh
index 9a444e3df..b385975f1 100755
--- a/dags/openshift_nightlies/scripts/run_benchmark.sh
+++ b/dags/openshift_nightlies/scripts/run_benchmark.sh
@@ -72,7 +72,7 @@ EOF
 }
 
 display_install_data(){    
-    if [[ $PLATFORM == "rosa" ]]; then
+    if [[ $PLATFORM == "rosa" || $PLATFORM == "rogcp" ]]; then
         IFS='@'
         # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
         read -ra newarr <<< "$InstallUUID"
diff --git a/dags/openshift_nightlies/tasks/benchmarks/e2e.py b/dags/openshift_nightlies/tasks/benchmarks/e2e.py
index b6429430a..801cd5b16 100644
--- a/dags/openshift_nightlies/tasks/benchmarks/e2e.py
+++ b/dags/openshift_nightlies/tasks/benchmarks/e2e.py
@@ -112,7 +112,7 @@ def _add_indexer(self, benchmark):
         benchmark >> indexer 
 
     def _get_benchmark(self, benchmark):
-        env = {**self.env, **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}}
+        env = {**self.env, "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}', **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}}
         # Fetch variables from a secret with the name <DAG_NAME>-<TASK_NAME>
         task_variables = var_loader.get_secret(f"{self.dag.dag_id}-{benchmark['name']}", True, False)
         env.update(task_variables)
@@ -128,7 +128,7 @@ def _get_benchmark(self, benchmark):
                 retries=0,
                 trigger_rule=benchmark.get("trigger_rule", "all_success"),
                 dag=self.dag,
-                env={ **env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'},
+                env=env,
                 do_xcom_push=True,
                 execution_timeout=timedelta(seconds=21600),
                 executor_config=self.exec_config
diff --git a/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py b/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py
index c1820f82c..aa41d06a3 100644
--- a/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py
+++ b/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py
@@ -26,6 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
     def _get_task(self, operation="install", trigger_rule="all_success"):
         self._setup_task(operation=operation)
         command=f"{constants.root_dag_dir}/scripts/install/ocm_gcp.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}"
+        env={ **self.env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'}
         return BashOperator(
             task_id=f"{operation}",
             depends_on_past=False,
@@ -34,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"):
             dag=self.dag,
             trigger_rule=trigger_rule,
             executor_config=self.exec_config,
-            env=self.env
+            env=env
         )
diff --git a/dags/openshift_nightlies/tasks/install/rosa/rosa.py b/dags/openshift_nightlies/tasks/install/rosa/rosa.py
index f8668a88e..21200070d 100644
--- a/dags/openshift_nightlies/tasks/install/rosa/rosa.py
+++ b/dags/openshift_nightlies/tasks/install/rosa/rosa.py
@@ -26,7 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
     def _get_task(self, operation="install", trigger_rule="all_success"):
         self._setup_task(operation=operation)
         command=f"{constants.root_dag_dir}/scripts/install/rosa.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}"
-
+        env={ **self.env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'}
         return BashOperator(
             task_id=f"{operation}",
             depends_on_past=False,
@@ -35,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"):
             dag=self.dag,
             trigger_rule=trigger_rule,
             executor_config=self.exec_config,
-            env={ **self.env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'}
+            env=env
         )

From 151cfa70e9ded32198f1271e9d1e1e2a9513161a Mon Sep 17 00:00:00 2001
From: Harshith-umesh <harshith.umesh.nat@gmail.com>
Date: Mon, 27 Jun 2022 18:00:00 +0530
Subject: [PATCH 3/4] passing vars for rogcp dags

---
 dags/openshift_nightlies/scripts/install/ocm_gcp.sh | 11 +++++++++++
 dags/openshift_nightlies/scripts/install/rosa.sh    |  1 -
 dags/openshift_nightlies/scripts/run_benchmark.sh   |  2 --
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
index 960686f4b..210c01cbf 100755
--- a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
+++ b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
@@ -103,6 +103,16 @@ postinstall(){
     gcloud compute firewall-rules create ${NETWORK_NAME}-hostnet --network ${NETWORK_NAME} --priority 105 --description 'scale-ci allow tcp,udp hostnetwork tests' --rules tcp:32768-60999,udp:32768-60999 --action allow
 }
 
+display_install_data(){
+    IFS='@'
+    # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
+    read -ra newarr <<< "$InstallUUID"
+    clustername="${newarr[0]}"
+    installuuid="${newarr[1]}"
+    echo "Cluster Name = ${clustername}"
+    echo "Install UUID = ${installuuid}"
+}
+
 cleanup(){
     ocm delete cluster $(_get_cluster_id ${CLUSTER_NAME})
     ocm logout
@@ -148,5 +158,6 @@ if [[ "$operation" == "install" ]]; then
 
 elif [[ "$operation" == "cleanup" ]]; then
     printf "Running Cleanup Steps"
+    display_install_data
     cleanup
 fi
diff --git a/dags/openshift_nightlies/scripts/install/rosa.sh b/dags/openshift_nightlies/scripts/install/rosa.sh
index 086156c0a..9dd6ef069 100755
--- a/dags/openshift_nightlies/scripts/install/rosa.sh
+++ b/dags/openshift_nightlies/scripts/install/rosa.sh
@@ -342,7 +342,6 @@ display_install_data(){
     IFS='@'
     # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
     read -ra newarr <<< "$InstallUUID"
-
     clustername="${newarr[0]}"
     installuuid="${newarr[1]}"
     echo "Cluster Name = ${clustername}"
diff --git a/dags/openshift_nightlies/scripts/run_benchmark.sh b/dags/openshift_nightlies/scripts/run_benchmark.sh
index b385975f1..fab304ca1 100755
--- a/dags/openshift_nightlies/scripts/run_benchmark.sh
+++ b/dags/openshift_nightlies/scripts/run_benchmark.sh
@@ -76,7 +76,6 @@ display_install_data(){
         IFS='@'
         # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
         read -ra newarr <<< "$InstallUUID"
-
         clustername="${newarr[0]}"
         installuuid="${newarr[1]}"
         echo "Cluster Name = ${clustername}"
@@ -84,7 +83,6 @@ display_install_data(){
     fi
 }
 
-
 export UUID=$(uuidgen | head -c8)-$AIRFLOW_CTX_TASK_ID-$(date '+%Y%m%d')
 echo "############################################"
 echo "# Benchmark UUID: ${UUID}"

From c9449c314bb7f5a522286b8fa60fc204c17eaa92 Mon Sep 17 00:00:00 2001
From: Harshith-umesh <harshith.umesh.nat@gmail.com>
Date: Mon, 27 Jun 2022 20:36:23 +0530
Subject: [PATCH 4/4] pass install_vars to following dag tasks

---
 dags/openshift_nightlies/scripts/install/ocm_gcp.sh   | 2 +-
 dags/openshift_nightlies/scripts/install/rosa.sh      | 2 +-
 dags/openshift_nightlies/scripts/run_benchmark.sh     | 2 +-
 dags/openshift_nightlies/tasks/benchmarks/e2e.py      | 2 +-
 dags/openshift_nightlies/tasks/install/rogcp/rogcp.py | 2 +-
 dags/openshift_nightlies/tasks/install/rosa/rosa.py   | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
index 210c01cbf..106a7e45f 100755
--- a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
+++ b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh
@@ -106,7 +106,7 @@ postinstall(){
 display_install_data(){
     IFS='@'
     # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
-    read -ra newarr <<< "$InstallUUID"
+    read -ra newarr <<< "$Install_vars"
     clustername="${newarr[0]}"
     installuuid="${newarr[1]}"
     echo "Cluster Name = ${clustername}"
diff --git a/dags/openshift_nightlies/scripts/install/rosa.sh b/dags/openshift_nightlies/scripts/install/rosa.sh
index 9dd6ef069..98d79a860 100755
--- a/dags/openshift_nightlies/scripts/install/rosa.sh
+++ b/dags/openshift_nightlies/scripts/install/rosa.sh
@@ -341,7 +341,7 @@ EOF
 display_install_data(){
     IFS='@'
     # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
-    read -ra newarr <<< "$InstallUUID"
+    read -ra newarr <<< "$Install_vars"
     clustername="${newarr[0]}"
     installuuid="${newarr[1]}"
     echo "Cluster Name = ${clustername}"
diff --git a/dags/openshift_nightlies/scripts/run_benchmark.sh b/dags/openshift_nightlies/scripts/run_benchmark.sh
index fab304ca1..f00f40984 100755
--- a/dags/openshift_nightlies/scripts/run_benchmark.sh
+++ b/dags/openshift_nightlies/scripts/run_benchmark.sh
@@ -75,7 +75,7 @@ display_install_data(){
     if [[ $PLATFORM == "rosa" || $PLATFORM == "rogcp" ]]; then
         IFS='@'
         # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
-        read -ra newarr <<< "$InstallUUID"
+        read -ra newarr <<< "$Install_vars"
         clustername="${newarr[0]}"
         installuuid="${newarr[1]}"
         echo "Cluster Name = ${clustername}"
diff --git a/dags/openshift_nightlies/tasks/benchmarks/e2e.py b/dags/openshift_nightlies/tasks/benchmarks/e2e.py
index 801cd5b16..2c88558a6 100644
--- a/dags/openshift_nightlies/tasks/benchmarks/e2e.py
+++ b/dags/openshift_nightlies/tasks/benchmarks/e2e.py
@@ -112,7 +112,7 @@ def _add_indexer(self, benchmark):
         benchmark >> indexer 
 
     def _get_benchmark(self, benchmark):
-        env = {**self.env, "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}', **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}}
+        env = {**self.env, "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}', **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}}
         # Fetch variables from a secret with the name <DAG_NAME>-<TASK_NAME>
         task_variables = var_loader.get_secret(f"{self.dag.dag_id}-{benchmark['name']}", True, False)
         env.update(task_variables)
diff --git a/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py b/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py
index aa41d06a3..b29b757d0 100644
--- a/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py
+++ b/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py
@@ -26,7 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
     def _get_task(self, operation="install", trigger_rule="all_success"):
         self._setup_task(operation=operation)
         command=f"{constants.root_dag_dir}/scripts/install/ocm_gcp.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}"
-        env={ **self.env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'}
+        env={ **self.env , "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}'}
         return BashOperator(
             task_id=f"{operation}",
             depends_on_past=False,
diff --git a/dags/openshift_nightlies/tasks/install/rosa/rosa.py b/dags/openshift_nightlies/tasks/install/rosa/rosa.py
index 21200070d..d3aaea238 100644
--- a/dags/openshift_nightlies/tasks/install/rosa/rosa.py
+++ b/dags/openshift_nightlies/tasks/install/rosa/rosa.py
@@ -26,7 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
     def _get_task(self, operation="install", trigger_rule="all_success"):
         self._setup_task(operation=operation)
         command=f"{constants.root_dag_dir}/scripts/install/rosa.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}"
-        env={ **self.env , "InstallUUID": '{{ ti.xcom_pull(task_ids="install")}}'}
+        env={ **self.env , "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}'}
         return BashOperator(
             task_id=f"{operation}",
             depends_on_past=False,