Enable google drive support (#899)

Author: ebattat

benchmark_runner/common/google_drive/google_drive_operations.py

@@ -1,6 +1,6 @@
 
 import os
-import logging
+import json
 
 from googleapiclient.discovery import build
 from google.oauth2.credentials import Credentials
@@ -20,14 +20,14 @@ class GoogleDriveOperations:
     1. credentials.json - Obtain from the "Google Cloud Console" under "APIs & Services" > "Credentials" > "Create Credentials" and select "OAuth 2.0 Client ID".
     2. token.json - This is generated automatically. If a browser is not available, copy the token from a machine with a browser.
     """
-    def __init__(self, google_drive_path: str, credentials_path: str, token_path: str, shared_drive_id: str):
+    def __init__(self, google_drive_path: str, google_drive_credentials: str, google_drive_token: str, google_drive_shared_drive_id: str):
         """
         Initializes GoogleDriveOperations with authentication.
         """
         self._google_drive_path = google_drive_path
-        self._credentials_path = credentials_path
-        self._token_path = token_path
-        self._shared_drive_id = shared_drive_id
+        self._google_drive_credentials = google_drive_credentials
+        self._google_drive_token = google_drive_token
+        self._google_drive_shared_drive_id = google_drive_shared_drive_id
         self.creds = self.authenticate()
         self.service = build('drive', 'v3', credentials=self.creds)
 
@@ -37,16 +37,23 @@ def authenticate(self):
         :return: Credentials object
         """
         creds = None
-        if os.path.exists(self._token_path):
-            creds = Credentials.from_authorized_user_file(self._token_path, SCOPES)
+
+        # Load credentials from the token string if it exists
+        if self._google_drive_token:
+            creds = Credentials.from_authorized_user_info(json.loads(self._google_drive_token), SCOPES)
+
+        # Check if the credentials are invalid or expired
         if not creds or not creds.valid:
             if creds and creds.expired and creds.refresh_token:
                 creds.refresh(Request())
             else:
-                flow = InstalledAppFlow.from_client_secrets_file(self._credentials_path, SCOPES)
+                # Load client secrets from the credentials string and run the local server for auth
+                flow = InstalledAppFlow.from_client_config(json.loads(self._google_drive_credentials), SCOPES)
                 creds = flow.run_local_server(port=0)
-            with open(self._token_path, 'w') as token_file:
-                token_file.write(creds.to_json())
+
+            # Save the new token to the variable as a string
+            self._google_drive_token = creds.to_json()
+
         return creds
 
     def list_folders_and_files_in_shared_drive(self, shared_drive_id):

benchmark_runner/main/environment_variables.py

@@ -201,9 +201,9 @@ def __init__(self):
 
         # Google drive
         self._environment_variables_dict['google_drive_path'] = EnvironmentVariables.get_env('GOOGLE_DRIVE_PATH', '')
-        self._environment_variables_dict['credentials_path'] = EnvironmentVariables.get_env('CREDENTIALS_PATH', '')
-        self._environment_variables_dict['token_path'] = EnvironmentVariables.get_env('TOKEN_PATH', '')
-        self._environment_variables_dict['shared_drive_id'] = EnvironmentVariables.get_env('SHARED_DRIVE_ID', '')
+        self._environment_variables_dict['google_drive_credentials'] = EnvironmentVariables.get_env('GOOGLE_DRIVE_CREDENTIALS', '')
+        self._environment_variables_dict['google_drive_token'] = EnvironmentVariables.get_env('GOOGLE_DRIVE_TOKEN', '')
+        self._environment_variables_dict['google_drive_shared_drive_id'] = EnvironmentVariables.get_env('GOOGLE_DRIVE_SHARED_DRIVE_ID', '')
 
         # Grafana
         self._environment_variables_dict['grafana_url'] = EnvironmentVariables.get_env('GRAFANA_URL', '')

benchmark_runner/workloads/bootstorm_vm.py

@@ -210,10 +210,10 @@ def _verify_vm_ssh(self):
                 self._oc.generate_cnv_must_gather(destination_path=self._run_artifacts_path, cnv_version=self._cnv_version)
                 self._oc.generate_odf_must_gather(destination_path=self._run_artifacts_path, odf_version=self._odf_version)
                 # google drive
-                if self._shared_drive_id:
+                if self._google_drive_shared_drive_id:
                     self.upload_run_artifacts_to_google_drive()
                 # s3
-                elif self._endpoint_url and not self._shared_drive_id:
+                elif self._endpoint_url and not self._google_drive_shared_drive_id:
                     self.upload_run_artifacts_to_s3()
                 # local
                 else:

benchmark_runner/workloads/workloads_operations.py

@@ -110,13 +110,14 @@ def __init__(self):
             self._windows_os = os.path.splitext(file_name)[0]
         # google drive
         self._google_drive_path = self._environment_variables_dict.get('google_drive_path', '')
-        self._credentials_path = self._environment_variables_dict.get('credentials_path', '')
-        self._token_path = self._environment_variables_dict.get('token_path', '')
-        self._shared_drive_id = self._environment_variables_dict.get('shared_drive_id', '')
-        self._google_drive_operation = GoogleDriveOperations(google_drive_path=self._google_drive_path,
-                                                             credentials_path=self._credentials_path,
-                                                             token_path=self._token_path,
-                                                             shared_drive_id=self._shared_drive_id)
+        self._google_drive_credentials = self._environment_variables_dict.get('google_drive_credentials', '')
+        self._google_drive_token = self._environment_variables_dict.get('google_drive_token', '')
+        self._google_drive_shared_drive_id = self._environment_variables_dict.get('google_drive_shared_drive_id', '')
+        if self._google_drive_path:
+            self._google_drive_operation = GoogleDriveOperations(google_drive_path=self._google_drive_path,
+                                                                 google_drive_credentials=self._google_drive_credentials,
+                                                                 google_drive_token=self._google_drive_token,
+                                                                 google_drive_shared_drive_id=self._google_drive_shared_drive_id)
 
     def _get_workload_file_name(self, workload):
         """
@@ -360,7 +361,7 @@ def get_run_artifacts_google_drive(self):
         """
         workload = self._workload.replace('_', '-')
         run_artifacts_hierarchy = self._get_run_artifacts_hierarchy(workload_name=workload)
-        return self._google_drive_operation.get_drive_folder_url(folder_path=run_artifacts_hierarchy, parent_folder_id=self._shared_drive_id)
+        return self._google_drive_operation.get_drive_folder_url(folder_path=run_artifacts_hierarchy, parent_folder_id=self._google_drive_shared_drive_id)
 
     @logger_time_stamp
     def upload_run_artifacts_to_google_drive(self):
@@ -377,7 +378,7 @@ def upload_run_artifacts_to_google_drive(self):
         upload_file = f"{workload_file_name}.tar.gz"
         self._google_drive_operation.upload_file_to_folder(file_path=tar_run_artifacts_path,
                                                            folder_path=str(run_artifacts_hierarchy),
-                                                           parent_folder_id=self._shared_drive_id)
+                                                           parent_folder_id=self._google_drive_shared_drive_id)
 
 
     def __get_metadata(self, kind: str = None, status: str = None, result: dict = None) -> dict:

jenkins/PerfCI_Chaos/04_PerfCI_Chaos_Upgrade_OpenShift_Deployment/Jenkinsfile

@@ -7,6 +7,7 @@ pipeline {
         LSO_VERSION = '4.15'
         ODF_VERSION = '4.15'
         CNV_VERSION = '4.15'
+        WAIT_FOR_UPGRADE_VERSION = '4.15.23'
         CONTACT_EMAIL1 = credentials('perfci_contact_email1')
         PROVISION_PRIVATE_KEY_FILE = credentials('perfci_provision_private_key_file')
         PROVISION_IP = credentials('perfci_provision_ip')
@@ -39,6 +40,10 @@ pipeline {
         IBM_SECRET_ACCESS_KEY = credentials('perfci_ibm_secret_access_key')
         IBM_BUCKET = credentials('perfci_ibm_bucket')
         IBM_KEY = credentials('perfci_ibm_key')
+        GOOGLE_DRIVE_PATH = credentials('perfci_google_drive_path')
+        GOOGLE_DRIVE_CREDENTIALS = credentials('perfci_google_drive_credentials')
+        GOOGLE_DRIVE_TOKEN = credentials('perfci_google_drive_token')
+        GOOGLE_DRIVE_SHARED_DRIVE_ID = credentials('perfci_google_drive_shared_drive_id')
         RUN_ARTIFACTS_URL = credentials('perfci_run_artifacts_url')
         REDIS = credentials('perfci_redis')
         WORKER_DISK_IDS = credentials('perfci_worker_disk_ids')
@@ -212,6 +217,7 @@ END
                                     sh """
                                         sudo podman run --rm -t \
                                         -e WORKLOAD='${workload}' \
+                                        -e WAIT_FOR_UPGRADE_VERSION='${WAIT_FOR_UPGRADE_VERSION}' \
                                         -e KUBEADMIN_PASSWORD='${KUBEADMIN_PASSWORD}' \
                                         -e PIN_NODE_BENCHMARK_OPERATOR='${PIN_NODE_BENCHMARK_OPERATOR}' \
                                         -e PIN_NODE1='${PIN_NODE1}' \
@@ -224,6 +230,10 @@ END
                                         -e IBM_SECRET_ACCESS_KEY='${IBM_SECRET_ACCESS_KEY}' \
                                         -e IBM_BUCKET='${IBM_BUCKET}' \
                                         -e IBM_KEY='${IBM_KEY}' \
+                                        -e GOOGLE_DRIVE_PATH='${GOOGLE_DRIVE_PATH}' \
+                                        -e GOOGLE_DRIVE_CREDENTIALS='${GOOGLE_DRIVE_CREDENTIALS}' \
+                                        -e GOOGLE_DRIVE_TOKEN='${GOOGLE_DRIVE_TOKEN}' \
+                                        -e GOOGLE_DRIVE_SHARED_DRIVE_ID='${GOOGLE_DRIVE_SHARED_DRIVE_ID}' \
                                         -e RUN_ARTIFACTS_URL='${RUN_ARTIFACTS_URL}' \
                                         -e BUILD_VERSION='${build_version}' \
                                         -e RUN_TYPE='${RUN_TYPE}' \
@@ -265,7 +275,7 @@ END
                  }
                 }
             }
-        } // WORKLOADS Deployment
+        } // WORKLOADS Verifications
     }
         }
     } // stages

tests/unittest/benchmark_runner/common/google_drive/mock_google_drive.py

@@ -21,7 +21,7 @@ def decorated(*args, **kwargs):
         # Patch the GoogleDriveOperations to use the mock service
         with patch('benchmark_runner.common.google_drive.google_drive_operations.GoogleDriveOperations.authenticate', return_value=None):
             with patch('benchmark_runner.common.google_drive.google_drive_operations.build', return_value=mock_service):
-                google_drive_operations = GoogleDriveOperations(google_drive_path='google_drive_path', credentials_path='credentials.json', token_path='token.json', shared_drive_id='shared_drive_id')
+                google_drive_operations = GoogleDriveOperations(google_drive_path='google_drive_path', google_drive_credentials='google_drive_credentials', google_drive_token='google_drive_token', google_drive_shared_drive_id='google_drive_shared_drive_id')
                 google_drive_operations.service = mock_service
 
                 return func(google_drive_operations, mock_service, *args, **kwargs)