diff --git a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh index 630a7b4af..106a7e45f 100755 --- a/dags/openshift_nightlies/scripts/install/ocm_gcp.sh +++ b/dags/openshift_nightlies/scripts/install/ocm_gcp.sh @@ -103,6 +103,16 @@ postinstall(){ gcloud compute firewall-rules create ${NETWORK_NAME}-hostnet --network ${NETWORK_NAME} --priority 105 --description 'scale-ci allow tcp,udp hostnetwork tests' --rules tcp:32768-60999,udp:32768-60999 --action allow } +display_install_data(){ + IFS='@' + # Read the xcom pushed by install task and store it into an array based on '@' as delimiter + read -ra newarr <<< "$Install_vars" + clustername="${newarr[0]}" + installuuid="${newarr[1]}" + echo "Cluster Name = ${clustername}" + echo "Install UUID = ${installuuid}" +} + cleanup(){ ocm delete cluster $(_get_cluster_id ${CLUSTER_NAME}) ocm logout @@ -144,8 +154,10 @@ if [[ "$operation" == "install" ]]; then printf "INFO: Cluster ${CLUSTER_NAME} already installed but not ready, exiting..." exit 1 fi + echo "${CLUSTER_NAME}@${UUID}" elif [[ "$operation" == "cleanup" ]]; then printf "Running Cleanup Steps" + display_install_data cleanup fi diff --git a/dags/openshift_nightlies/scripts/install/rosa.sh b/dags/openshift_nightlies/scripts/install/rosa.sh index f3ee70f0d..98d79a860 100755 --- a/dags/openshift_nightlies/scripts/install/rosa.sh +++ b/dags/openshift_nightlies/scripts/install/rosa.sh @@ -338,6 +338,16 @@ EOF return 0 } +display_install_data(){ + IFS='@' + # Read the xcom pushed by install task and store it into an array based on '@' as delimiter + read -ra newarr <<< "$Install_vars" + clustername="${newarr[0]}" + installuuid="${newarr[1]}" + echo "Cluster Name = ${clustername}" + echo "Install UUID = ${installuuid}" +} + cleanup(){ if [[ $INSTALL_METHOD == "osd" ]]; then ocm delete cluster $(_get_cluster_id ${CLUSTER_NAME}) @@ -377,9 +387,11 @@ if [[ "$operation" == "install" ]]; then printf "INFO: Cluster ${CLUSTER_NAME} already installed but not ready, exiting..." exit 1 fi + echo "${CLUSTER_NAME}@${UUID}" elif [[ "$operation" == "cleanup" ]]; then printf "Running Cleanup Steps" + display_install_data cleanup index_metadata rosa logout diff --git a/dags/openshift_nightlies/scripts/run_benchmark.sh b/dags/openshift_nightlies/scripts/run_benchmark.sh index 1a17f7158..f00f40984 100755 --- a/dags/openshift_nightlies/scripts/run_benchmark.sh +++ b/dags/openshift_nightlies/scripts/run_benchmark.sh @@ -70,6 +70,19 @@ EOF exit 1 fi } + +display_install_data(){ + if [[ $PLATFORM == "rosa" || $PLATFORM == "rogcp" ]]; then + IFS='@' + # Read the xcom pushed by install task and store it into an array based on '@' as delimiter + read -ra newarr <<< "$Install_vars" + clustername="${newarr[0]}" + installuuid="${newarr[1]}" + echo "Cluster Name = ${clustername}" + echo "Install UUID = ${installuuid}" + fi +} + export UUID=$(uuidgen | head -c8)-$AIRFLOW_CTX_TASK_ID-$(date '+%Y%m%d') echo "############################################" echo "# Benchmark UUID: ${UUID}" @@ -80,6 +93,7 @@ if [[ $PLATFORM == "baremetal" ]]; then run_baremetal_benchmark echo $UUID else + display_install_data setup cd /home/airflow/workspace/e2e-benchmarking/workloads/$workload diff --git a/dags/openshift_nightlies/tasks/benchmarks/e2e.py b/dags/openshift_nightlies/tasks/benchmarks/e2e.py index 9bfbbd46f..2c88558a6 100644 --- a/dags/openshift_nightlies/tasks/benchmarks/e2e.py +++ b/dags/openshift_nightlies/tasks/benchmarks/e2e.py @@ -112,7 +112,7 @@ def _add_indexer(self, benchmark): benchmark >> indexer def _get_benchmark(self, benchmark): - env = {**self.env, **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}} + env = {**self.env, "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}', **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}} # Fetch variables from a secret with the name - task_variables = var_loader.get_secret(f"{self.dag.dag_id}-{benchmark['name']}", True, False) env.update(task_variables) diff --git a/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py b/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py index c1820f82c..b29b757d0 100644 --- a/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py +++ b/dags/openshift_nightlies/tasks/install/rogcp/rogcp.py @@ -26,6 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): def _get_task(self, operation="install", trigger_rule="all_success"): self._setup_task(operation=operation) command=f"{constants.root_dag_dir}/scripts/install/ocm_gcp.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}" + env={ **self.env , "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}'} return BashOperator( task_id=f"{operation}", depends_on_past=False, @@ -34,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"): dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, - env=self.env + env=env ) diff --git a/dags/openshift_nightlies/tasks/install/rosa/rosa.py b/dags/openshift_nightlies/tasks/install/rosa/rosa.py index cb1e0ea1b..d3aaea238 100644 --- a/dags/openshift_nightlies/tasks/install/rosa/rosa.py +++ b/dags/openshift_nightlies/tasks/install/rosa/rosa.py @@ -26,7 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): def _get_task(self, operation="install", trigger_rule="all_success"): self._setup_task(operation=operation) command=f"{constants.root_dag_dir}/scripts/install/rosa.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}" - + env={ **self.env , "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}'} return BashOperator( task_id=f"{operation}", depends_on_past=False, @@ -35,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"): dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, - env=self.env + env=env ) diff --git a/dags/openshift_nightlies/tasks/utils/final_dag_status.py b/dags/openshift_nightlies/tasks/utils/final_dag_status.py index 41b759239..19c90b39a 100644 --- a/dags/openshift_nightlies/tasks/utils/final_dag_status.py +++ b/dags/openshift_nightlies/tasks/utils/final_dag_status.py @@ -8,6 +8,7 @@ def final_status(**kwargs): + ti = kwargs['ti'] failed_tasks=[] for task_instance in kwargs['dag_run'].get_task_instances(): if "index" in task_instance.task_id: