From aae7869cf06b5a6d1d4b90b039d1359df730635b Mon Sep 17 00:00:00 2001 From: dingzhaohan Date: Wed, 18 Jun 2025 16:34:32 +0800 Subject: [PATCH 1/5] update bohrium sdk openapi --- dpdispatcher/contexts/openapi_context.py | 55 +++++++++++++++++++----- dpdispatcher/machines/openapi.py | 31 +++++++++---- 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/dpdispatcher/contexts/openapi_context.py b/dpdispatcher/contexts/openapi_context.py index 9251ce48..644684b7 100644 --- a/dpdispatcher/contexts/openapi_context.py +++ b/dpdispatcher/contexts/openapi_context.py @@ -1,14 +1,13 @@ import os import shutil import uuid - +import glob +from zipfile import ZipFile import tqdm try: - from bohriumsdk.client import Client - from bohriumsdk.job import Job - from bohriumsdk.storage import Storage - from bohriumsdk.util import Util + from bohrium import Bohrium + from bohrium.resources import Job, Tiefblue except ModuleNotFoundError: found_bohriumsdk = False else: @@ -22,6 +21,33 @@ os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/" ) +def unzip_file(zip_file, out_dir="./"): + obj = ZipFile(zip_file, "r") + for item in obj.namelist(): + obj.extract(item, out_dir) + +def zip_file_list(root_path, zip_filename, file_list=[]): + out_zip_file = os.path.join(root_path, zip_filename) + # print('debug: file_list', file_list) + zip_obj = ZipFile(out_zip_file, "w") + for f in file_list: + matched_files = os.path.join(root_path, f) + for ii in glob.glob(matched_files): + # print('debug: matched_files:ii', ii) + if os.path.isdir(ii): + arcname = os.path.relpath(ii, start=root_path) + zip_obj.write(ii, arcname) + for root, dirs, files in os.walk(ii): + for file in files: + filename = os.path.join(root, file) + arcname = os.path.relpath(filename, start=root_path) + # print('debug: filename:arcname:root_path', filename, arcname, root_path) + zip_obj.write(filename, arcname) + else: + arcname = os.path.relpath(ii, start=root_path) + zip_obj.write(ii, arcname) + zip_obj.close() + return out_zip_file class OpenAPIContext(BaseContext): def __init__( @@ -40,10 +66,19 @@ def __init__( self.init_remote_root = remote_root self.temp_local_root = os.path.abspath(local_root) self.remote_profile = remote_profile - self.client = Client() - self.storage = Storage(client=self.client) + access_key = remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) + project_id = remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) + if access_key is None: + raise ValueError( + "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" + ) + if project_id is None: + raise ValueError( + "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'" + ) + self.client = Bohrium(access_key=access_key, project_id=project_id) + self.storage = Tiefblue() self.job = Job(client=self.client) - self.util = Util() self.jgid = None @classmethod @@ -97,7 +132,7 @@ def upload_job(self, job, common_files=None): for file in task.forward_files: upload_file_list.append(os.path.join(task.task_work_path, file)) - upload_zip = Util.zip_file_list( + upload_zip = zip_file_list( self.local_root, zip_task_file, file_list=upload_file_list ) project_id = self.remote_profile.get("project_id", 0) @@ -189,7 +224,7 @@ def download(self, submission): ): continue self.storage.download_from_url(info["resultUrl"], target_result_zip) - Util.unzip_file(target_result_zip, out_dir=self.local_root) + unzip_file(target_result_zip, out_dir=self.local_root) self._backup(self.local_root, target_result_zip) self._clean_backup( self.local_root, keep_backup=self.remote_profile.get("keep_backup", True) diff --git a/dpdispatcher/machines/openapi.py b/dpdispatcher/machines/openapi.py index 83ba165d..25af01d1 100644 --- a/dpdispatcher/machines/openapi.py +++ b/dpdispatcher/machines/openapi.py @@ -1,14 +1,12 @@ import os import shutil import time - +from zipfile import ZipFile from dpdispatcher.utils.utils import customized_script_header_template try: - from bohriumsdk.client import Client - from bohriumsdk.job import Job - from bohriumsdk.storage import Storage - from bohriumsdk.util import Util + from bohrium import Bohrium + from bohrium.resources import Job, Tiefblue except ModuleNotFoundError: found_bohriumsdk = False else: @@ -22,6 +20,11 @@ #!/bin/bash -l """ +def unzip_file(zip_file, out_dir="./"): + obj = ZipFile(zip_file, "r") + for item in obj.namelist(): + obj.extract(item, out_dir) + class OpenAPI(Machine): def __init__(self, context): @@ -35,9 +38,20 @@ def __init__(self, context): self.grouped = self.remote_profile.get("grouped", True) self.retry_count = self.remote_profile.get("retry_count", 3) self.ignore_exit_code = context.remote_profile.get("ignore_exit_code", True) - self.client = Client() + + access_key = self.remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) + project_id = self.remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) + if access_key is None: + raise ValueError( + "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" + ) + if project_id is None: + raise ValueError( + "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'" + ) + self.client = Bohrium(access_key=access_key, project_id=project_id) + self.storage = Tiefblue() self.job = Job(client=self.client) - self.storage = Storage(client=self.client) self.group_id = None def gen_script(self, job): @@ -102,7 +116,6 @@ def do_submit(self, job): } if job.job_state == JobStatus.unsubmitted: openapi_params["job_id"] = job.job_id - data = self.job.insert(**openapi_params) job.job_id = data.get("jobId", 0) # type: ignore @@ -170,7 +183,7 @@ def _download_job(self, job): result_filename = job_hash + "_back.zip" target_result_zip = os.path.join(self.context.local_root, result_filename) self.storage.download_from_url(job_url, target_result_zip) - Util.unzip_file(target_result_zip, out_dir=self.context.local_root) + unzip_file(target_result_zip, out_dir=self.context.local_root) try: os.makedirs(os.path.join(self.context.local_root, "backup"), exist_ok=True) shutil.move( From 0a81a197e74dac79fb23f9284737f1e392ef8ae6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 08:49:23 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dpdispatcher/contexts/openapi_context.py | 18 +++++++++++++++--- dpdispatcher/machines/openapi.py | 14 ++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/dpdispatcher/contexts/openapi_context.py b/dpdispatcher/contexts/openapi_context.py index 644684b7..d9f12001 100644 --- a/dpdispatcher/contexts/openapi_context.py +++ b/dpdispatcher/contexts/openapi_context.py @@ -1,8 +1,9 @@ +import glob import os import shutil import uuid -import glob from zipfile import ZipFile + import tqdm try: @@ -21,11 +22,13 @@ os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/" ) + def unzip_file(zip_file, out_dir="./"): obj = ZipFile(zip_file, "r") for item in obj.namelist(): obj.extract(item, out_dir) + def zip_file_list(root_path, zip_filename, file_list=[]): out_zip_file = os.path.join(root_path, zip_filename) # print('debug: file_list', file_list) @@ -49,6 +52,7 @@ def zip_file_list(root_path, zip_filename, file_list=[]): zip_obj.close() return out_zip_file + class OpenAPIContext(BaseContext): def __init__( self, @@ -66,8 +70,16 @@ def __init__( self.init_remote_root = remote_root self.temp_local_root = os.path.abspath(local_root) self.remote_profile = remote_profile - access_key = remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) - project_id = remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) + access_key = ( + remote_profile.get("access_key", None) + or os.getenv("BOHRIUM_ACCESS_KEY", None) + or os.getenv("ACCESS_KEY", None) + ) + project_id = ( + remote_profile.get("project_id", None) + or os.getenv("BOHRIUM_PROJECT_ID", None) + or os.getenv("PROJECT_ID", None) + ) if access_key is None: raise ValueError( "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" diff --git a/dpdispatcher/machines/openapi.py b/dpdispatcher/machines/openapi.py index 25af01d1..192b9c03 100644 --- a/dpdispatcher/machines/openapi.py +++ b/dpdispatcher/machines/openapi.py @@ -2,6 +2,7 @@ import shutil import time from zipfile import ZipFile + from dpdispatcher.utils.utils import customized_script_header_template try: @@ -20,6 +21,7 @@ #!/bin/bash -l """ + def unzip_file(zip_file, out_dir="./"): obj = ZipFile(zip_file, "r") for item in obj.namelist(): @@ -39,8 +41,16 @@ def __init__(self, context): self.retry_count = self.remote_profile.get("retry_count", 3) self.ignore_exit_code = context.remote_profile.get("ignore_exit_code", True) - access_key = self.remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) - project_id = self.remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) + access_key = ( + self.remote_profile.get("access_key", None) + or os.getenv("BOHRIUM_ACCESS_KEY", None) + or os.getenv("ACCESS_KEY", None) + ) + project_id = ( + self.remote_profile.get("project_id", None) + or os.getenv("BOHRIUM_PROJECT_ID", None) + or os.getenv("PROJECT_ID", None) + ) if access_key is None: raise ValueError( "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" From e82f431c66abb42b323fd205e1bed6a76cd74015 Mon Sep 17 00:00:00 2001 From: dingzhaohan Date: Fri, 20 Jun 2025 14:17:45 +0800 Subject: [PATCH 3/5] support app-key --- dpdispatcher/contexts/openapi_context.py | 3 ++- dpdispatcher/machines/openapi.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dpdispatcher/contexts/openapi_context.py b/dpdispatcher/contexts/openapi_context.py index 644684b7..2eb2e62f 100644 --- a/dpdispatcher/contexts/openapi_context.py +++ b/dpdispatcher/contexts/openapi_context.py @@ -68,6 +68,7 @@ def __init__( self.remote_profile = remote_profile access_key = remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) project_id = remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) + app_key = remote_profile.get("app_key", None) or os.getenv("BOHRIUM_APP_KEY", None) or os.getenv("APP_KEY", None) if access_key is None: raise ValueError( "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" @@ -76,7 +77,7 @@ def __init__( raise ValueError( "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'" ) - self.client = Bohrium(access_key=access_key, project_id=project_id) + self.client = Bohrium(access_key=access_key, project_id=project_id, app_key=app_key) self.storage = Tiefblue() self.job = Job(client=self.client) self.jgid = None diff --git a/dpdispatcher/machines/openapi.py b/dpdispatcher/machines/openapi.py index 25af01d1..db90d788 100644 --- a/dpdispatcher/machines/openapi.py +++ b/dpdispatcher/machines/openapi.py @@ -41,6 +41,7 @@ def __init__(self, context): access_key = self.remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) project_id = self.remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) + app_key = self.remote_profile.get("app_key", None) or os.getenv("BOHRIUM_APP_KEY", None) or os.getenv("APP_KEY", None) if access_key is None: raise ValueError( "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" @@ -49,7 +50,7 @@ def __init__(self, context): raise ValueError( "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'" ) - self.client = Bohrium(access_key=access_key, project_id=project_id) + self.client = Bohrium(access_key=access_key, project_id=project_id, app_key=app_key) self.storage = Tiefblue() self.job = Job(client=self.client) self.group_id = None From 2f110f1eb438544fec2351beb39790017485a465 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 05:03:08 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dpdispatcher/contexts/openapi_context.py | 22 ++++++++++++++++++---- dpdispatcher/machines/openapi.py | 22 ++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/dpdispatcher/contexts/openapi_context.py b/dpdispatcher/contexts/openapi_context.py index ec3ee2b7..c1f85e31 100644 --- a/dpdispatcher/contexts/openapi_context.py +++ b/dpdispatcher/contexts/openapi_context.py @@ -70,9 +70,21 @@ def __init__( self.init_remote_root = remote_root self.temp_local_root = os.path.abspath(local_root) self.remote_profile = remote_profile - access_key = remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) - project_id = remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) - app_key = remote_profile.get("app_key", None) or os.getenv("BOHRIUM_APP_KEY", None) or os.getenv("APP_KEY", None) + access_key = ( + remote_profile.get("access_key", None) + or os.getenv("BOHRIUM_ACCESS_KEY", None) + or os.getenv("ACCESS_KEY", None) + ) + project_id = ( + remote_profile.get("project_id", None) + or os.getenv("BOHRIUM_PROJECT_ID", None) + or os.getenv("PROJECT_ID", None) + ) + app_key = ( + remote_profile.get("app_key", None) + or os.getenv("BOHRIUM_APP_KEY", None) + or os.getenv("APP_KEY", None) + ) if access_key is None: raise ValueError( "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" @@ -81,7 +93,9 @@ def __init__( raise ValueError( "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'" ) - self.client = Bohrium(access_key=access_key, project_id=project_id, app_key=app_key) + self.client = Bohrium( + access_key=access_key, project_id=project_id, app_key=app_key + ) self.storage = Tiefblue() self.job = Job(client=self.client) self.jgid = None diff --git a/dpdispatcher/machines/openapi.py b/dpdispatcher/machines/openapi.py index 96ef5aa7..3073c2f9 100644 --- a/dpdispatcher/machines/openapi.py +++ b/dpdispatcher/machines/openapi.py @@ -41,9 +41,21 @@ def __init__(self, context): self.retry_count = self.remote_profile.get("retry_count", 3) self.ignore_exit_code = context.remote_profile.get("ignore_exit_code", True) - access_key = self.remote_profile.get("access_key", None) or os.getenv("BOHRIUM_ACCESS_KEY", None) or os.getenv("ACCESS_KEY", None) - project_id = self.remote_profile.get("project_id", None) or os.getenv("BOHRIUM_PROJECT_ID", None) or os.getenv("PROJECT_ID", None) - app_key = self.remote_profile.get("app_key", None) or os.getenv("BOHRIUM_APP_KEY", None) or os.getenv("APP_KEY", None) + access_key = ( + self.remote_profile.get("access_key", None) + or os.getenv("BOHRIUM_ACCESS_KEY", None) + or os.getenv("ACCESS_KEY", None) + ) + project_id = ( + self.remote_profile.get("project_id", None) + or os.getenv("BOHRIUM_PROJECT_ID", None) + or os.getenv("PROJECT_ID", None) + ) + app_key = ( + self.remote_profile.get("app_key", None) + or os.getenv("BOHRIUM_APP_KEY", None) + or os.getenv("APP_KEY", None) + ) if access_key is None: raise ValueError( "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'" @@ -52,7 +64,9 @@ def __init__(self, context): raise ValueError( "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'" ) - self.client = Bohrium(access_key=access_key, project_id=project_id, app_key=app_key) + self.client = Bohrium( + access_key=access_key, project_id=project_id, app_key=app_key + ) self.storage = Tiefblue() self.job = Job(client=self.client) self.group_id = None From 42711123c61acee4b5fcb799745e863aaf0a104d Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Tue, 24 Jun 2025 17:58:38 +0800 Subject: [PATCH 5/5] print detailed import error --- dpdispatcher/contexts/openapi_context.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dpdispatcher/contexts/openapi_context.py b/dpdispatcher/contexts/openapi_context.py index c1f85e31..8346fcd5 100644 --- a/dpdispatcher/contexts/openapi_context.py +++ b/dpdispatcher/contexts/openapi_context.py @@ -9,10 +9,12 @@ try: from bohrium import Bohrium from bohrium.resources import Job, Tiefblue -except ModuleNotFoundError: +except ModuleNotFoundError as e: found_bohriumsdk = False + import_bohrium_error = e else: found_bohriumsdk = True + import_bohrium_error = None from dpdispatcher.base_context import BaseContext from dpdispatcher.dlog import dlog @@ -65,7 +67,7 @@ def __init__( if not found_bohriumsdk: raise ModuleNotFoundError( "bohriumsdk not installed. Install dpdispatcher with `pip install dpdispatcher[bohrium]`" - ) + ) from import_bohrium_error self.init_local_root = local_root self.init_remote_root = remote_root self.temp_local_root = os.path.abspath(local_root)