From b41454531c55c08e93e21b13da84520a80fb7502 Mon Sep 17 00:00:00 2001 From: Patrie Date: Tue, 9 May 2023 17:39:19 -0400 Subject: [PATCH 01/16] Added acceptable formats helper functions --- biosimulators_utils/config.py | 128 +++++++++++++++++++++++++++++----- 1 file changed, 109 insertions(+), 19 deletions(-) diff --git a/biosimulators_utils/config.py b/biosimulators_utils/config.py index 3e530a9a..b33b47a7 100644 --- a/biosimulators_utils/config.py +++ b/biosimulators_utils/config.py @@ -5,17 +5,26 @@ :Copyright: 2020, Center for Reproducible Biomedical Modeling :License: MIT """ - -from .omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, OmexMetadataSchema -from .report.data_model import ReportFormat # noqa: F401 -from .viz.data_model import VizFormat # noqa: F401 -from kisao import AlgorithmSubstitutionPolicy # noqa: F401 -import appdirs import enum import os +import platform +from types import FunctionType +from typing import Dict, List, Union +from kisao import AlgorithmSubstitutionPolicy # noqa: F401 +from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel +from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, OmexMetadataSchema +from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 +from biosimulators_utils.viz.data_model import VizFormat # noqa: F401 +import appdirs __all__ = ['Config', 'get_config', 'Colors', 'get_app_dirs'] +CURRENT_PLATFORM = platform.system() +try: + assert CURRENT_PLATFORM == "Darwin" + DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.python +except AssertionError as e: + DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c DEFAULT_OMEX_METADATA_INPUT_FORMAT = OmexMetadataInputFormat.rdfxml DEFAULT_OMEX_METADATA_OUTPUT_FORMAT = OmexMetadataOutputFormat.rdfxml_abbrev DEFAULT_OMEX_METADATA_SCHEMA = OmexMetadataSchema.biosimulations @@ -65,6 +74,7 @@ class Config(object): BIOSIMULATIONS_API_AUDIENCE (:obj:`str`): audience for the BioSimulations API VERBOSE (:obj:`bool`): whether to display the detailed output of the execution of each task DEBUG (:obj:`bool`): whether to raise exceptions rather than capturing them + STDOUT_LEVEL (:obj:`StandardOutputErrorCapturerLevel`): level at which to log the output """ def __init__(self, @@ -82,8 +92,8 @@ def __init__(self, COLLECT_COMBINE_ARCHIVE_RESULTS=False, COLLECT_SED_DOCUMENT_RESULTS=False, SAVE_PLOT_DATA=True, - REPORT_FORMATS=[ReportFormat.h5], - VIZ_FORMATS=[VizFormat.pdf], + REPORT_FORMATS=None, + VIZ_FORMATS=None, H5_REPORTS_PATH=DEFAULT_H5_REPORTS_PATH, REPORTS_PATH=DEFAULT_REPORTS_PATH, PLOTS_PATH=DEFAULT_PLOTS_PATH, @@ -96,7 +106,9 @@ def __init__(self, BIOSIMULATIONS_API_AUTH_ENDPOINT=DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT, BIOSIMULATIONS_API_AUDIENCE=DEFAULT_BIOSIMULATIONS_API_AUDIENCE, VERBOSE=False, - DEBUG=False): + DEBUG=False, + STDOUT_LEVEL=DEFAULT_STDOUT_LEVEL, + **CUSTOM_SETTINGS): """ Args: OMEX_METADATA_INPUT_FORMAT (:obj:`OmexMetadataInputFormat`, optional): format to validate OMEX Metadata files against @@ -117,7 +129,7 @@ def __init__(self, COLLECT_SED_DOCUMENT_RESULTS (:obj:`bool`, optional): whether to assemble an in memory data structure with all of the simulation results of SED documents SAVE_PLOT_DATA (:obj:`bool`, optional): whether to save data for plots alongside data for reports in CSV/HDF5 files - REPORT_FORMATS (:obj:`list` of :obj:`str`, optional): default formats to generate reports in + REPORT_FORMATS (:obj:`list` of :obj:`str`, optional): default formats to generate reports in-->defaults to 'csv' VIZ_FORMATS (:obj:`list` of :obj:`str`, optional): default formats to generate plots in H5_REPORTS_PATH (:obj:`str`, optional): path to save reports in HDF5 format relative to base output directory REPORTS_PATH (:obj:`str`, optional): path to save zip archive of reports relative to base output directory @@ -132,6 +144,7 @@ def __init__(self, BIOSIMULATIONS_API_AUDIENCE (:obj:`str`, optional): audience for the BioSimulations API VERBOSE (:obj:`bool`, optional): whether to display the detailed output of the execution of each task DEBUG (:obj:`bool`, optional): whether to raise exceptions rather than capturing them + STDOUT_LEVEL (:obj:`StandardOutputErrorCapturerLevel`): level at which to log the output """ self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT self.OMEX_METADATA_OUTPUT_FORMAT = OMEX_METADATA_OUTPUT_FORMAT @@ -147,8 +160,8 @@ def __init__(self, self.COLLECT_COMBINE_ARCHIVE_RESULTS = COLLECT_COMBINE_ARCHIVE_RESULTS self.COLLECT_SED_DOCUMENT_RESULTS = COLLECT_SED_DOCUMENT_RESULTS self.SAVE_PLOT_DATA = SAVE_PLOT_DATA - self.REPORT_FORMATS = REPORT_FORMATS - self.VIZ_FORMATS = VIZ_FORMATS + self.REPORT_FORMATS = REPORT_FORMATS or [ReportFormat.csv] + self.VIZ_FORMATS = VIZ_FORMATS or [VizFormat.pdf] self.H5_REPORTS_PATH = H5_REPORTS_PATH self.REPORTS_PATH = REPORTS_PATH self.PLOTS_PATH = PLOTS_PATH @@ -162,23 +175,65 @@ def __init__(self, self.BIOSIMULATIONS_API_AUDIENCE = BIOSIMULATIONS_API_AUDIENCE self.VERBOSE = VERBOSE self.DEBUG = DEBUG + self.STDOUT_LEVEL = STDOUT_LEVEL + self.CUSTOM_SETTINGS = self.__getcustomsettings(CUSTOM_SETTINGS) + + def __getcustomsettings(self, settings: Dict = None): + for key in settings.keys(): + setattr(self, key, settings[key]) + return self -def get_config(): - """ Get the configuration - +def get_config(report_format: str = None, + viz_format: str = None, + acceptable_report_formats: Union[List[str], ReportFormat] = ReportFormat, + acceptable_viz_formats: Union[List[str], VizFormat] = VizFormat, + *_default_format_settings): + """ Get the configuration based on specified optional settings. Handles sets default values for + `report_format` and `viz_format` if these respective variables are empty. + + Args: + :str:`report_format`: output format for reports. Defaults to `None` + + :str:`viz_format`: output format for visualizations. Defaults to `None` + + :Union:`acceptable_report_formats`: acceptable formats for output report data. Defaults to `biosimulators_utils.report.data_model.ReportFormat` + + :Union:`acceptable_viz_formats`: acceptable formats for output viz data. Defaults to `biosimulators_utils.viz.data_model.VizFormat` + Returns: :obj:`Config`: configuration """ - report_formats = os.environ.get('REPORT_FORMATS', 'h5').strip() + + if not _default_format_settings: #get + _default_format_settings = ('csv', 'pdf') #set + + user_report_format = verify_formats( + report_format, + acceptable_report_formats, + _default_format_settings[0] + ) + + user_viz_format = verify_formats( + viz_format, + acceptable_viz_formats, + _default_format_settings[1] + ) + + report_formats = os.environ.get('REPORT_FORMATS', user_report_format).strip() + if report_formats: - report_formats = [ReportFormat(format.strip().lower()) for format in report_formats.split(',')] + report_formats = [ + ReportFormat(format.strip().lower()) for format in report_formats.split(',') + ] else: report_formats = [] - viz_formats = os.environ.get('VIZ_FORMATS', 'pdf').strip() + viz_formats = os.environ.get('VIZ_FORMATS', user_viz_format).strip() if viz_formats: - viz_formats = [VizFormat(format.strip().lower()) for format in viz_formats.split(',')] + viz_formats = [ + VizFormat(format.strip().lower()) for format in viz_formats.split(',') + ] else: viz_formats = [] @@ -216,6 +271,7 @@ def get_config(): BIOSIMULATIONS_API_AUDIENCE=os.environ.get('BIOSIMULATIONS_API_AUDIENCE', DEFAULT_BIOSIMULATIONS_API_AUDIENCE), VERBOSE=os.environ.get('VERBOSE', '1').lower() in ['1', 'true'], DEBUG=os.environ.get('DEBUG', '0').lower() in ['1', 'true'], + STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL) ) @@ -244,3 +300,37 @@ def get_app_dirs(): :obj:`appdirs.AppDirs`: application directories """ return appdirs.AppDirs("BioSimulatorsUtils", "BioSimulatorsTeam") + + +def verify_formats(format_type: str, acceptable_format: enum.Enum, default: str): + + def verify_format(format_type, acceptable_format): + acceptable_formats = acceptable_formats(acceptable_format) + if format_type not in acceptable_formats: + print( + f'''Sorry, you must enter one of the following acceptable formats: + {acceptable_formats}. \nSetting to default format: {default}''' + ) + return False + else: + return True + + return default if not verify_format(format_type, acceptable_format)\ + else format_type + + +def acceptable_viz_formats(): + return acceptable_formats(VizFormat) + +def acceptable_report_formats(): + return acceptable_formats(ReportFormat) + +def acceptable_formats(acceptable_formats: enum.Enum): + return [v.value for v in acceptable_formats] + + + + + + + From 12b92fbf28e231e7bef836af45adc4df4571785a Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 14:42:58 -0400 Subject: [PATCH 02/16] added absolute imports --- biosimulators_utils/log/data_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/biosimulators_utils/log/data_model.py b/biosimulators_utils/log/data_model.py index 0e0434e6..45920edd 100644 --- a/biosimulators_utils/log/data_model.py +++ b/biosimulators_utils/log/data_model.py @@ -6,7 +6,7 @@ :License: MIT """ -from ..config import get_config +from biosimulators_utils.config import get_config import enum import os import yaml From 32e1fe3997e13d4dc4822175147c28e416aa52f3 Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:17:03 -0400 Subject: [PATCH 03/16] added validation --- biosimulators_utils/warnings.py | 2 +- biosimulators_utils/wip_config.py | 316 ++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 biosimulators_utils/wip_config.py diff --git a/biosimulators_utils/warnings.py b/biosimulators_utils/warnings.py index ba19ec47..b5298f8b 100644 --- a/biosimulators_utils/warnings.py +++ b/biosimulators_utils/warnings.py @@ -6,7 +6,7 @@ :License: MIT """ -from .config import Colors +from biosimulators_utils.config import Colors import termcolor import warnings diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py new file mode 100644 index 00000000..3fe7ff07 --- /dev/null +++ b/biosimulators_utils/wip_config.py @@ -0,0 +1,316 @@ +import enum +import os +import json +import platform +from datetime import datetime +from typing import Dict, List, Union +from kisao import AlgorithmSubstitutionPolicy # noqa: F401 +#from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel +from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, \ + OmexMetadataSchema +from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 +from biosimulators_utils.viz.data_model import VizFormat # noqa: F401 +import appdirs + +__all__ = [ + 'Config', 'get_config', 'Colors', + 'get_app_dirs', 'acceptable_report_formats', + 'acceptable_viz_formats' +] + +class StandardOutputErrorCapturerLevel(str, enum.Enum): + python = 'python' + c = 'c' + + +CURRENT_PLATFORM = platform.system() +try: + assert CURRENT_PLATFORM == "Darwin" + DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.python +except AssertionError as e: + DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c +DEFAULT_OMEX_METADATA_INPUT_FORMAT = OmexMetadataInputFormat.rdfxml +DEFAULT_OMEX_METADATA_OUTPUT_FORMAT = OmexMetadataOutputFormat.rdfxml_abbrev +DEFAULT_OMEX_METADATA_SCHEMA = OmexMetadataSchema.biosimulations +DEFAULT_ALGORITHM_SUBSTITUTION_POLICY = AlgorithmSubstitutionPolicy.SIMILAR_VARIABLES +DEFAULT_H5_REPORTS_PATH = 'reports.h5' +DEFAULT_REPORTS_PATH = 'reports.zip' +DEFAULT_PLOTS_PATH = 'plots.zip' +DEFAULT_LOG_PATH = 'log.yml' +DEFAULT_BIOSIMULATORS_API_ENDPOINT = 'https://api.biosimulators.org/' +DEFAULT_BIOSIMULATIONS_API_ENDPOINT = 'https://api.biosimulations.org/' +DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT = 'https://auth.biosimulations.org/oauth/token' +DEFAULT_BIOSIMULATIONS_API_AUDIENCE = 'api.biosimulations.org' + + +class Config(object): + def __init__(self, + OMEX_METADATA_INPUT_FORMAT=DEFAULT_OMEX_METADATA_INPUT_FORMAT, + OMEX_METADATA_OUTPUT_FORMAT=DEFAULT_OMEX_METADATA_OUTPUT_FORMAT, + OMEX_METADATA_SCHEMA=DEFAULT_OMEX_METADATA_SCHEMA, + VALIDATE_OMEX_MANIFESTS=True, + VALIDATE_SEDML=True, + VALIDATE_SEDML_MODELS=True, + VALIDATE_IMPORTED_MODEL_FILES=True, + VALIDATE_OMEX_METADATA=True, + VALIDATE_IMAGES=True, + VALIDATE_RESULTS=True, + ALGORITHM_SUBSTITUTION_POLICY=DEFAULT_ALGORITHM_SUBSTITUTION_POLICY, + COLLECT_COMBINE_ARCHIVE_RESULTS=False, + COLLECT_SED_DOCUMENT_RESULTS=False, + SAVE_PLOT_DATA=True, + REPORT_FORMATS=None, + VIZ_FORMATS=None, + H5_REPORTS_PATH=DEFAULT_H5_REPORTS_PATH, + REPORTS_PATH=DEFAULT_REPORTS_PATH, + PLOTS_PATH=DEFAULT_PLOTS_PATH, + BUNDLE_OUTPUTS=True, + KEEP_INDIVIDUAL_OUTPUTS=True, + LOG=True, + LOG_PATH=DEFAULT_LOG_PATH, + BIOSIMULATORS_API_ENDPOINT=DEFAULT_BIOSIMULATORS_API_ENDPOINT, + BIOSIMULATIONS_API_ENDPOINT=DEFAULT_BIOSIMULATIONS_API_ENDPOINT, + BIOSIMULATIONS_API_AUTH_ENDPOINT=DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT, + BIOSIMULATIONS_API_AUDIENCE=DEFAULT_BIOSIMULATIONS_API_AUDIENCE, + VERBOSE=False, + DEBUG=False, + STDOUT_LEVEL=DEFAULT_STDOUT_LEVEL, + **CUSTOM_SETTINGS): + + self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT + self.OMEX_METADATA_OUTPUT_FORMAT = OMEX_METADATA_OUTPUT_FORMAT + self.OMEX_METADATA_SCHEMA = OMEX_METADATA_SCHEMA + self.VALIDATE_OMEX_MANIFESTS = VALIDATE_OMEX_MANIFESTS + self.VALIDATE_SEDML = VALIDATE_SEDML + self.VALIDATE_SEDML_MODELS = VALIDATE_SEDML_MODELS + self.VALIDATE_IMPORTED_MODEL_FILES = VALIDATE_IMPORTED_MODEL_FILES + self.VALIDATE_OMEX_METADATA = VALIDATE_OMEX_METADATA + self.VALIDATE_IMAGES = VALIDATE_IMAGES + self.VALIDATE_RESULTS = VALIDATE_RESULTS + self.ALGORITHM_SUBSTITUTION_POLICY = ALGORITHM_SUBSTITUTION_POLICY + self.COLLECT_COMBINE_ARCHIVE_RESULTS = COLLECT_COMBINE_ARCHIVE_RESULTS + self.COLLECT_SED_DOCUMENT_RESULTS = COLLECT_SED_DOCUMENT_RESULTS + self.SAVE_PLOT_DATA = SAVE_PLOT_DATA + self.REPORT_FORMATS = [ReportFormat.csv] if not REPORT_FORMATS else REPORT_FORMATS + self.VIZ_FORMATS = [VizFormat.pdf] if not VIZ_FORMATS else VIZ_FORMATS + self.H5_REPORTS_PATH = H5_REPORTS_PATH + self.REPORTS_PATH = REPORTS_PATH + self.PLOTS_PATH = PLOTS_PATH + self.BUNDLE_OUTPUTS = BUNDLE_OUTPUTS + self.KEEP_INDIVIDUAL_OUTPUTS = KEEP_INDIVIDUAL_OUTPUTS + self.LOG = LOG + self.LOG_PATH = LOG_PATH + self.BIOSIMULATORS_API_ENDPOINT = BIOSIMULATORS_API_ENDPOINT + self.BIOSIMULATIONS_API_ENDPOINT = BIOSIMULATIONS_API_ENDPOINT + self.BIOSIMULATIONS_API_AUTH_ENDPOINT = BIOSIMULATIONS_API_AUTH_ENDPOINT + self.BIOSIMULATIONS_API_AUDIENCE = BIOSIMULATIONS_API_AUDIENCE + self.VERBOSE = VERBOSE + self.DEBUG = DEBUG + self.STDOUT_LEVEL = STDOUT_LEVEL + self.CUSTOM_SETTINGS = self.__getcustomsettings(CUSTOM_SETTINGS) + if "EASY_LOG" in CUSTOM_SETTINGS: + self.logger = self.easy_log() + + def __getcustomsettings(self, settings: Dict = None): + for key in settings.keys(): + setattr(self, key, settings[key]) + return self + + def easy_log(self): + return EasyLog(os.getcwd()) + + +def get_config(report_format: str = None, + viz_format: str = None, + acceptable_report_formats: Union[List[str], ReportFormat] = ReportFormat, + acceptable_viz_formats: Union[List[str], VizFormat] = VizFormat, + easy_log: bool = False, + *_default_format_settings) -> Config: + + if not _default_format_settings: # get + _default_format_settings = ('csv', 'pdf') # set + + user_report_format = verify_formats( + report_format, + acceptable_report_formats, + _default_format_settings[0] + ) + + user_viz_format = verify_formats( + viz_format, + acceptable_viz_formats, + _default_format_settings[1] + ) + + print(user_viz_format, user_report_format) + + '''report_formats = os.environ.get('REPORT_FORMATS', user_report_format).strip() + + if report_formats: + report_formats = [ + ReportFormat(format.strip().lower()) for format in report_formats.split(',') + ] + else: + report_formats = [] + + viz_formats = os.environ.get('VIZ_FORMATS', user_viz_format).strip() + if viz_formats: + viz_formats = [ + VizFormat(format.strip().lower()) for format in viz_formats.split(',') + ] + else: + viz_formats = [] + + return Config( + OMEX_METADATA_INPUT_FORMAT=OmexMetadataInputFormat(os.environ.get( + 'OMEX_METADATA_INPUT_FORMAT', DEFAULT_OMEX_METADATA_INPUT_FORMAT)), + OMEX_METADATA_OUTPUT_FORMAT=OmexMetadataOutputFormat(os.environ.get( + 'OMEX_METADATA_OUTPUT_FORMAT', DEFAULT_OMEX_METADATA_OUTPUT_FORMAT)), + OMEX_METADATA_SCHEMA=OmexMetadataSchema(os.environ.get( + 'OMEX_METADATA_SCHEMA', DEFAULT_OMEX_METADATA_SCHEMA)), + VALIDATE_OMEX_MANIFESTS=os.environ.get('VALIDATE_OMEX_MANIFESTS', '1').lower() in ['1', 'true'], + VALIDATE_SEDML=os.environ.get('VALIDATE_SEDML', '1').lower() in ['1', 'true'], + VALIDATE_SEDML_MODELS=os.environ.get('VALIDATE_SEDML_MODELS', '1').lower() in ['1', 'true'], + VALIDATE_IMPORTED_MODEL_FILES=os.environ.get('VALIDATE_IMPORTED_MODEL_FILES', '1').lower() in ['1', 'true'], + VALIDATE_OMEX_METADATA=os.environ.get('VALIDATE_OMEX_METADATA', '1').lower() in ['1', 'true'], + VALIDATE_IMAGES=os.environ.get('VALIDATE_IMAGES', '1').lower() in ['1', 'true'], + VALIDATE_RESULTS=os.environ.get('VALIDATE_RESULTS', '1').lower() in ['1', 'true'], + ALGORITHM_SUBSTITUTION_POLICY=AlgorithmSubstitutionPolicy(os.environ.get( + 'ALGORITHM_SUBSTITUTION_POLICY', DEFAULT_ALGORITHM_SUBSTITUTION_POLICY)), + COLLECT_COMBINE_ARCHIVE_RESULTS=os.environ.get('COLLECT_COMBINE_ARCHIVE_RESULTS', '0').lower() in ['1', 'true'], + COLLECT_SED_DOCUMENT_RESULTS=os.environ.get('COLLECT_SED_DOCUMENT_RESULTS', '0').lower() in ['1', 'true'], + SAVE_PLOT_DATA=os.environ.get('SAVE_PLOT_DATA', '1').lower() in ['1', 'true'], + REPORT_FORMATS=report_formats, + VIZ_FORMATS=viz_formats, + H5_REPORTS_PATH=os.environ.get('H5_REPORTS_PATH', DEFAULT_H5_REPORTS_PATH), + REPORTS_PATH=os.environ.get('REPORTS_PATH', DEFAULT_REPORTS_PATH), + PLOTS_PATH=os.environ.get('PLOTS_PATH', DEFAULT_PLOTS_PATH), + BUNDLE_OUTPUTS=os.environ.get('BUNDLE_OUTPUTS', '1').lower() in ['1', 'true'], + KEEP_INDIVIDUAL_OUTPUTS=os.environ.get('KEEP_INDIVIDUAL_OUTPUTS', '1').lower() in ['1', 'true'], + LOG=os.environ.get('LOG', '1').lower() in ['1', 'true'], + LOG_PATH=os.environ.get('LOG_PATH', DEFAULT_LOG_PATH), + BIOSIMULATORS_API_ENDPOINT=os.environ.get('BIOSIMULATORS_API_ENDPOINT', DEFAULT_BIOSIMULATORS_API_ENDPOINT), + BIOSIMULATIONS_API_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_ENDPOINT), + BIOSIMULATIONS_API_AUTH_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_AUTH_ENDPOINT', + DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT), + BIOSIMULATIONS_API_AUDIENCE=os.environ.get('BIOSIMULATIONS_API_AUDIENCE', DEFAULT_BIOSIMULATIONS_API_AUDIENCE), + VERBOSE=os.environ.get('VERBOSE', '1').lower() in ['1', 'true'], + DEBUG=os.environ.get('DEBUG', '0').lower() in ['1', 'true'], + STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL), + EASY_LOG=None if not easy_log else easy_log + )''' + +get_config() + +Colors = enum.Enum('Colors', + { + 'queued': 'cyan', + 'success': 'blue', + 'succeeded': 'blue', + 'running': 'green', + 'pass': 'green', + 'passed': 'green', + 'failure': 'red', + 'failed': 'red', + 'skip': 'magenta', + 'skipped': 'magenta', + 'warning': 'yellow', + 'warned': 'yellow', + }, + type=str) + + +def get_app_dirs(): + """ Get the directories for the application + + Returns: + :obj:`appdirs.AppDirs`: application directories + """ + return appdirs.AppDirs("BioSimulatorsUtils", "BioSimulatorsTeam") + + +def acceptable_viz_formats(): + return get_acceptable_formats(VizFormat) + + +def acceptable_report_formats(): + return get_acceptable_formats(ReportFormat) + + +def get_acceptable_formats(acceptable_formats: enum.Enum): + return [v.value for v in acceptable_formats] + + +def verify_formats(format_type: str, acceptable_format: enum.Enum, default: str): + def verify_format(form_type, acceptable_form): + acceptable_formats = get_acceptable_formats(acceptable_form) + if form_type not in acceptable_formats: + print( + f'''Sorry, you must enter one of the following acceptable formats: + {acceptable_formats}. \nSetting to default format: {default}''' + ) + return False + else: + return True + + return default if not verify_format(format_type, acceptable_format) \ + else format_type + + +class EasyLog: + def __init__(self, log_dir, fresh: bool = False): + self.working_file = __file__ + self._make_logdir(log_dir, fresh) + self.log = {} + self.index = list(range(len(self.log))) + + def __getsize__(self): + return len(self.log) + + def __getnow__(self): + return datetime.now().strftime("%d.%m.%Y..%H.%M.%S") + + def _make_logdir(self, log_dir: str, fresh_log: bool): + make_dir = lambda: os.os.mkdir(log_dir) if not os.path.exists(log_dir) else None + if fresh_log: + filepaths = [] + for root, _, files in os.walk(log_dir): + filepaths.append(os.path.join(root, f) for f in files) + for f in filepaths: + os.remove(f) + os.rmdir(log_dir) + make_dir() + else: + make_dir() + + def add_msg(self, message, function="none", status="none"): + size = self.__getsize__() + entry_number = 1 if size < 1 else size + now = self.__getnow__() + verify = lambda v: v != "none" + function = function.__name__ if verify(function) else function + status = str(status) + entry = f"""{now} | NOTES: {message} | CALLED FROM: {self.working_file} + | METHOD CALLED: {function} | STATUS: {status.upper()}""" + self.log[entry_number] = entry + return self.log + + def flush(self): + for n in self.index: + self.log.pop(n) + return self.log + + def write(self, log_fp: str = None): + if not log_fp: + now = self.__getnow__() + log_fp = f"log_{now}.json" + with open(log_fp, "w"): + json.dump(self.log, log_fp, indent=4) + + def flush_log(self, log_fp: str = None): + self.write(log_fp) + self.flush() + return + + + From 1477b6a0ea023637e63fa0031e9a3b15cbb3bda3 Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:32:46 -0400 Subject: [PATCH 04/16] added testing --- biosimulators_utils/wip_config.py | 6 ++-- tests/test_config.py | 51 ++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index 3fe7ff07..5645b99b 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -144,7 +144,7 @@ def get_config(report_format: str = None, print(user_viz_format, user_report_format) - '''report_formats = os.environ.get('REPORT_FORMATS', user_report_format).strip() + report_formats = os.environ.get('REPORT_FORMATS', user_report_format).strip() if report_formats: report_formats = [ @@ -198,9 +198,9 @@ def get_config(report_format: str = None, DEBUG=os.environ.get('DEBUG', '0').lower() in ['1', 'true'], STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL), EASY_LOG=None if not easy_log else easy_log - )''' + ) + -get_config() Colors = enum.Enum('Colors', { diff --git a/tests/test_config.py b/tests/test_config.py index fadb48d8..3b9548fb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,9 @@ -from biosimulators_utils.config import get_config, get_app_dirs -from biosimulators_utils.report.data_model import ReportFormat -from biosimulators_utils.viz.data_model import VizFormat from unittest import mock -import os import unittest +import os +from biosimulators_utils.config import get_config, get_app_dirs, Config +from biosimulators_utils.report.data_model import ReportFormat +from biosimulators_utils.viz.data_model import VizFormat class ConfigTestCase(unittest.TestCase): @@ -20,3 +20,46 @@ def test_get_config(self): def test_get_app_dirs(self): self.assertIn('BioSimulatorsUtils', get_app_dirs().user_data_dir) + + +class TestConfig(unittest.TestCase): + def test_init(self): + with mock.patch('appdirs.user_config_dir') as mock_app_dirs: + mock_app_dirs.return_value = '/fake/config/dir' + config = Config( + REPORT_FORMATS='csv', + VIZ_FORMATS='pdf', + CUSTOM_SETTINGS={'MY_CUSTOM_SETTING': 'my_custom_value'} + ) + self.assertEqual(config.OMEX_METADATA_INPUT_FORMAT, 'rdfxml') + self.assertEqual(config.OMEX_METADATA_OUTPUT_FORMAT, 'rdfxml_abbrev') + self.assertEqual(config.OMEX_METADATA_SCHEMA, 'biosimulations') + self.assertTrue(config.VALIDATE_OMEX_MANIFESTS) + self.assertTrue(config.VALIDATE_SEDML) + self.assertTrue(config.VALIDATE_SEDML_MODELS) + self.assertTrue(config.VALIDATE_IMPORTED_MODEL_FILES) + self.assertTrue(config.VALIDATE_OMEX_METADATA) + self.assertTrue(config.VALIDATE_IMAGES) + self.assertTrue(config.VALIDATE_RESULTS) + self.assertEqual(config.ALGORITHM_SUBSTITUTION_POLICY, 'similarVariables') + self.assertFalse(config.COLLECT_COMBINE_ARCHIVE_RESULTS) + self.assertFalse(config.COLLECT_SED_DOCUMENT_RESULTS) + self.assertTrue(config.SAVE_PLOT_DATA) + self.assertEqual(config.REPORT_FORMATS, [ReportFormat.h5.value]) + self.assertEqual(config.VIZ_FORMATS, [VizFormat.pdf.value]) + self.assertEqual(config.H5_REPORTS_PATH, 'reports.h5') + self.assertEqual(config.REPORTS_PATH, 'reports.zip') + self.assertEqual(config.PLOTS_PATH, 'plots.zip') + self.assertTrue(config.BUNDLE_OUTPUTS) + self.assertTrue(config.KEEP_INDIVIDUAL_OUTPUTS) + self.assertTrue(config.LOG) + self.assertEqual(config.LOG_PATH, 'log.yml') + self.assertEqual(config.BIOSIMULATORS_API_ENDPOINT, 'https://api.biosimulators.org/') + self.assertEqual(config.BIOSIMULATIONS_API_ENDPOINT, 'https://api.biosimulations.org/') + self.assertEqual(config.BIOSIMULATIONS_API_AUTH_ENDPOINT, 'https://auth.biosimulations.org/oauth/token') + self.assertEqual(config.BIOSIMULATIONS_API_AUDIENCE, 'api.biosimulations.org') + self.assertFalse(config.VERBOSE) + self.assertFalse(config.DEBUG) + self.assertEqual(config.STDOUT_LEVEL, 'python') + self.assertEqual(config.CUSTOM_SETTINGS, {'MY_CUSTOM_SETTING': 'my_custom_value'}) + self.assertEqual(config.logger.handlers[0].baseFilename, '/fake/config/dir/logs/app.log') From c648da97e287f404e0ecc719cd888e98245ac052 Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:37:11 -0400 Subject: [PATCH 05/16] minor adjustments --- biosimulators_utils/config.py | 133 +++++++++++++++++++------- biosimulators_utils/log/data_model.py | 3 +- biosimulators_utils/sedml/exec.py | 5 +- biosimulators_utils/warnings.py | 2 +- biosimulators_utils/wip_config.py | 8 +- 5 files changed, 108 insertions(+), 43 deletions(-) diff --git a/biosimulators_utils/config.py b/biosimulators_utils/config.py index b33b47a7..782fb105 100644 --- a/biosimulators_utils/config.py +++ b/biosimulators_utils/config.py @@ -5,19 +5,26 @@ :Copyright: 2020, Center for Reproducible Biomedical Modeling :License: MIT """ + import enum import os +import json import platform -from types import FunctionType +from datetime import datetime from typing import Dict, List, Union from kisao import AlgorithmSubstitutionPolicy # noqa: F401 from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel -from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, OmexMetadataSchema +from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, \ + OmexMetadataSchema from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 from biosimulators_utils.viz.data_model import VizFormat # noqa: F401 import appdirs -__all__ = ['Config', 'get_config', 'Colors', 'get_app_dirs'] +__all__ = [ + 'Config', 'get_config', 'Colors', + 'get_app_dirs', 'acceptable_report_formats', + 'acceptable_viz_formats' +] CURRENT_PLATFORM = platform.system() try: @@ -177,18 +184,24 @@ def __init__(self, self.DEBUG = DEBUG self.STDOUT_LEVEL = STDOUT_LEVEL self.CUSTOM_SETTINGS = self.__getcustomsettings(CUSTOM_SETTINGS) - + if "EASY_LOG" in self.CUSTOM_SETTINGS: + self.logger = self.easy_log() + def __getcustomsettings(self, settings: Dict = None): for key in settings.keys(): setattr(self, key, settings[key]) - return self + return self + + def easy_log(self): + return EasyLog(os.getcwd()) def get_config(report_format: str = None, viz_format: str = None, acceptable_report_formats: Union[List[str], ReportFormat] = ReportFormat, acceptable_viz_formats: Union[List[str], VizFormat] = VizFormat, - *_default_format_settings): + easy_log: bool = False, + *_default_format_settings) -> Config: """ Get the configuration based on specified optional settings. Handles sets default values for `report_format` and `viz_format` if these respective variables are empty. @@ -204,24 +217,24 @@ def get_config(report_format: str = None, Returns: :obj:`Config`: configuration """ - - if not _default_format_settings: #get - _default_format_settings = ('csv', 'pdf') #set - + + if not _default_format_settings: # get + _default_format_settings = ('csv', 'pdf') # set + user_report_format = verify_formats( - report_format, - acceptable_report_formats, + report_format, + acceptable_report_formats, _default_format_settings[0] ) - + user_viz_format = verify_formats( - viz_format, - acceptable_viz_formats, + viz_format, + acceptable_viz_formats, _default_format_settings[1] ) - + report_formats = os.environ.get('REPORT_FORMATS', user_report_format).strip() - + if report_formats: report_formats = [ ReportFormat(format.strip().lower()) for format in report_formats.split(',') @@ -267,11 +280,13 @@ def get_config(report_format: str = None, LOG_PATH=os.environ.get('LOG_PATH', DEFAULT_LOG_PATH), BIOSIMULATORS_API_ENDPOINT=os.environ.get('BIOSIMULATORS_API_ENDPOINT', DEFAULT_BIOSIMULATORS_API_ENDPOINT), BIOSIMULATIONS_API_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_ENDPOINT), - BIOSIMULATIONS_API_AUTH_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_AUTH_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT), + BIOSIMULATIONS_API_AUTH_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_AUTH_ENDPOINT', + DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT), BIOSIMULATIONS_API_AUDIENCE=os.environ.get('BIOSIMULATIONS_API_AUDIENCE', DEFAULT_BIOSIMULATIONS_API_AUDIENCE), VERBOSE=os.environ.get('VERBOSE', '1').lower() in ['1', 'true'], DEBUG=os.environ.get('DEBUG', '0').lower() in ['1', 'true'], - STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL) + STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL), + EASY_LOG=None if not easy_log else easy_log ) @@ -302,35 +317,85 @@ def get_app_dirs(): return appdirs.AppDirs("BioSimulatorsUtils", "BioSimulatorsTeam") +def acceptable_viz_formats(): + return get_acceptable_formats(VizFormat) + + +def acceptable_report_formats(): + return get_acceptable_formats(ReportFormat) + + +def get_acceptable_formats(acceptable_formats: enum.Enum): + return [v.value for v in acceptable_formats] + + def verify_formats(format_type: str, acceptable_format: enum.Enum, default: str): - - def verify_format(format_type, acceptable_format): - acceptable_formats = acceptable_formats(acceptable_format) - if format_type not in acceptable_formats: + def verify_format(form_type, acceptable_form): + acceptable_formats = get_acceptable_formats(acceptable_form) + if form_type not in acceptable_formats: print( f'''Sorry, you must enter one of the following acceptable formats: {acceptable_formats}. \nSetting to default format: {default}''' ) - return False + return False else: return True - - return default if not verify_format(format_type, acceptable_format)\ - else format_type + return default if not verify_format(format_type, acceptable_format) \ + else format_type -def acceptable_viz_formats(): - return acceptable_formats(VizFormat) -def acceptable_report_formats(): - return acceptable_formats(ReportFormat) +class EasyLog: + def __init__(self, log_dir, fresh: bool = False): + self.working_file = __file__ + self._make_logdir(log_dir, fresh) + self.log = {} + self.index = list(range(len(self.log))) -def acceptable_formats(acceptable_formats: enum.Enum): - return [v.value for v in acceptable_formats] + def __getsize__(self): + return len(self.log) - + def __getnow__(self): + return datetime.now().strftime("%d.%m.%Y..%H.%M.%S") + def _make_logdir(self, log_dir: str, fresh_log: bool): + make_dir = lambda: os.os.mkdir(log_dir) if not os.path.exists(log_dir) else None + if fresh_log: + filepaths = [] + for root, _, files in os.walk(log_dir): + filepaths.append(os.path.join(root, f) for f in files) + for f in filepaths: + os.remove(f) + os.rmdir(log_dir) + make_dir() + else: + make_dir() + def add_msg(self, message, function="none", status="none"): + size = self.__getsize__() + entry_number = 1 if size < 1 else size + now = self.__getnow__() + verify = lambda v: v != "none" + function = function.__name__ if verify(function) else function + status = str(status) + entry = f"""{now} | NOTES: {message} | CALLED FROM: {self.working_file} + | METHOD CALLED: {function} | STATUS: {status.upper()}""" + self.log[entry_number] = entry + return self.log + def flush(self): + for n in self.index: + self.log.pop(n) + return self.log + def write(self, log_fp: str = None): + if not log_fp: + now = self.__getnow__() + log_fp = f"log_{now}.json" + with open(log_fp, "w"): + json.dump(self.log, log_fp, indent=4) + def flush_log(self, log_fp: str = None): + self.write(log_fp) + self.flush() + return diff --git a/biosimulators_utils/log/data_model.py b/biosimulators_utils/log/data_model.py index 45920edd..76e4ea0e 100644 --- a/biosimulators_utils/log/data_model.py +++ b/biosimulators_utils/log/data_model.py @@ -5,7 +5,8 @@ :Copyright: 2020, Center for Reproducible Biomedical Modeling :License: MIT """ - +import sys +sys.path.insert(0, "C:/Users/apatrie/Desktop/wip-biosimulators-utils-repo/Biosimulators_utils") from biosimulators_utils.config import get_config import enum import os diff --git a/biosimulators_utils/sedml/exec.py b/biosimulators_utils/sedml/exec.py index 2e688fc9..f11a6141 100644 --- a/biosimulators_utils/sedml/exec.py +++ b/biosimulators_utils/sedml/exec.py @@ -50,7 +50,7 @@ def exec_sed_doc(task_executer, doc, working_dir, base_out_path, rel_out_path=None, apply_xml_model_changes=False, log=None, indent=0, pretty_print_modified_xml_models=False, - log_level=StandardOutputErrorCapturerLevel.c, + log_level=None, config=None): """ Execute the tasks specified in a SED document and generate the specified outputs @@ -104,6 +104,9 @@ def exec_task(task, variables, preprocessed_task=None, log=None, config=None, ** if not config: config = get_config() + if not log_level: + log_level = config.STDOUT_LEVEL + # process arguments if not isinstance(doc, SedDocument): doc = SedmlSimulationReader().run(doc, config=config) diff --git a/biosimulators_utils/warnings.py b/biosimulators_utils/warnings.py index b5298f8b..66893295 100644 --- a/biosimulators_utils/warnings.py +++ b/biosimulators_utils/warnings.py @@ -6,7 +6,7 @@ :License: MIT """ -from biosimulators_utils.config import Colors +from wip_config import Colors import termcolor import warnings diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index 5645b99b..777305c0 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Dict, List, Union from kisao import AlgorithmSubstitutionPolicy # noqa: F401 -#from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel +# from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, \ OmexMetadataSchema from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 @@ -18,6 +18,7 @@ 'acceptable_viz_formats' ] + class StandardOutputErrorCapturerLevel(str, enum.Enum): python = 'python' c = 'c' @@ -126,7 +127,6 @@ def get_config(report_format: str = None, acceptable_viz_formats: Union[List[str], VizFormat] = VizFormat, easy_log: bool = False, *_default_format_settings) -> Config: - if not _default_format_settings: # get _default_format_settings = ('csv', 'pdf') # set @@ -201,7 +201,6 @@ def get_config(report_format: str = None, ) - Colors = enum.Enum('Colors', { 'queued': 'cyan', @@ -311,6 +310,3 @@ def flush_log(self, log_fp: str = None): self.write(log_fp) self.flush() return - - - From 46ba80acf576873606c98637934613cbbca4183b Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:45:12 -0400 Subject: [PATCH 06/16] removed shadow alias --- biosimulators_utils/wip_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index 777305c0..2f4bc862 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -148,7 +148,7 @@ def get_config(report_format: str = None, if report_formats: report_formats = [ - ReportFormat(format.strip().lower()) for format in report_formats.split(',') + ReportFormat(f.strip().lower()) for f in report_formats.split(',') ] else: report_formats = [] @@ -156,7 +156,7 @@ def get_config(report_format: str = None, viz_formats = os.environ.get('VIZ_FORMATS', user_viz_format).strip() if viz_formats: viz_formats = [ - VizFormat(format.strip().lower()) for format in viz_formats.split(',') + VizFormat(f.strip().lower()) for f in viz_formats.split(',') ] else: viz_formats = [] From 898db2724e9032c7b1372850d4d61cbfbeb8b901 Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:46:51 -0400 Subject: [PATCH 07/16] minor linting --- biosimulators_utils/wip_config.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index 2f4bc862..8ab41389 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Dict, List, Union from kisao import AlgorithmSubstitutionPolicy # noqa: F401 -# from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel +from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, \ OmexMetadataSchema from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 @@ -19,11 +19,6 @@ ] -class StandardOutputErrorCapturerLevel(str, enum.Enum): - python = 'python' - c = 'c' - - CURRENT_PLATFORM = platform.system() try: assert CURRENT_PLATFORM == "Darwin" From 7c3b2e329720578785efbc8a51a1f87953c7c492 Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:48:40 -0400 Subject: [PATCH 08/16] minor linting --- biosimulators_utils/wip_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index 8ab41389..0143441a 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -277,15 +277,15 @@ def _make_logdir(self, log_dir: str, fresh_log: bool): else: make_dir() - def add_msg(self, message, function="none", status="none"): + def add_msg(self, message, func="none", status="none"): size = self.__getsize__() entry_number = 1 if size < 1 else size now = self.__getnow__() verify = lambda v: v != "none" - function = function.__name__ if verify(function) else function + func = func.__name__ if verify(func) else func status = str(status) entry = f"""{now} | NOTES: {message} | CALLED FROM: {self.working_file} - | METHOD CALLED: {function} | STATUS: {status.upper()}""" + | METHOD CALLED: {func} | STATUS: {status.upper()}""" self.log[entry_number] = entry return self.log From 51cd11246db9dc54817ad21890b5262c96a23dfb Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:49:44 -0400 Subject: [PATCH 09/16] minor linting --- biosimulators_utils/wip_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index 0143441a..c08fbb45 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -304,4 +304,3 @@ def write(self, log_fp: str = None): def flush_log(self, log_fp: str = None): self.write(log_fp) self.flush() - return From 042dda010874a3b5c177fcf8ccda5ec05d952b69 Mon Sep 17 00:00:00 2001 From: Patrie Date: Wed, 10 May 2023 15:56:42 -0400 Subject: [PATCH 10/16] minor linting for --- biosimulators_utils/wip_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py index c08fbb45..9aa81f7f 100644 --- a/biosimulators_utils/wip_config.py +++ b/biosimulators_utils/wip_config.py @@ -241,7 +241,7 @@ def verify_format(form_type, acceptable_form): if form_type not in acceptable_formats: print( f'''Sorry, you must enter one of the following acceptable formats: - {acceptable_formats}. \nSetting to default format: {default}''' + {acceptable_formats}.\nSetting to default format: {default}''' ) return False else: From 04350a0340c0ccace42929eeb62b47190c6e1952 Mon Sep 17 00:00:00 2001 From: "A.P" <105522603+AlexPatrie@users.noreply.github.com> Date: Thu, 11 May 2023 00:46:12 +0000 Subject: [PATCH 11/16] implemented config and typing --- biosimulators_utils/sedml/exec.py | 38 ++++++++++++++++++------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/biosimulators_utils/sedml/exec.py b/biosimulators_utils/sedml/exec.py index f11a6141..05696409 100644 --- a/biosimulators_utils/sedml/exec.py +++ b/biosimulators_utils/sedml/exec.py @@ -6,6 +6,18 @@ :License: MIT """ + +import copy +import datetime +import numpy +import os +import sys +import tempfile +import termcolor +import types # noqa: F401 +from lxml import etree # noqa: F401 +from types import FunctionType +from typing import List, Tuple, Union from ..config import get_config, Config, Colors # noqa: F401 from ..log.data_model import Status, SedDocumentLog, TaskLog, ReportLog, Plot2DLog, Plot3DLog, StandardOutputErrorCapturerLevel # noqa: F401 from ..log.utils import init_sed_document_log, StandardOutputErrorCapturer @@ -24,15 +36,7 @@ apply_changes_to_xml_model, get_first_last_models_executed_by_task, is_model_language_encoded_in_xml) from .warnings import NoTasksWarning, NoOutputsWarning, SedmlFeatureNotSupportedWarning -from lxml import etree # noqa: F401 -import copy -import datetime -import numpy -import os -import sys -import tempfile -import termcolor -import types # noqa: F401 + __all__ = [ @@ -47,11 +51,11 @@ ] -def exec_sed_doc(task_executer, doc, working_dir, base_out_path, rel_out_path=None, - apply_xml_model_changes=False, - log=None, indent=0, pretty_print_modified_xml_models=False, - log_level=None, - config=None): +def exec_sed_doc(task_executer: FunctionType, doc: Union[SedDocument, str], + working_dir: str, base_out_path: str, rel_out_path: str = None, + apply_xml_model_changes: bool = False, log: SedDocumentLog = None, + indent: int = 0, pretty_print_modified_xml_models: bool = False, + log_level: StandardOutputErrorCapturerLevel = None, config: Config = None): """ Execute the tasks specified in a SED document and generate the specified outputs Args: @@ -102,7 +106,7 @@ def exec_task(task, variables, preprocessed_task=None, log=None, config=None, ** * :obj:`SedDocumentLog`: log of the document """ if not config: - config = get_config() + config = get_config(easy_log=True) if not log_level: log_level = config.STDOUT_LEVEL @@ -758,7 +762,9 @@ def exec_plot_2d(plot, variable_results, base_out_path, rel_out_path, formats, t return status, data_gen_exceptions, task_contributes_to_plot -def exec_plot_3d(plot, variable_results, base_out_path, rel_out_path, formats, task, log): +def exec_plot_3d(plot: Plot3D, variable_results: VariableResults, + base_out_path: str, rel_out_path: str, formats: List[VizFormat], + task: Task, log: ReportLog) -> Tuple[Status, Exception, bool]: """ Execute a 3D plot, generating the surfaces which are available Args: From e07ccfc109887aa9d84702ef34e09c9ccc5cd716 Mon Sep 17 00:00:00 2001 From: "A.P" <105522603+AlexPatrie@users.noreply.github.com> Date: Thu, 11 May 2023 02:06:01 +0000 Subject: [PATCH 12/16] added docstrings --- biosimulators_utils/combine/exec.py | 15 +-- biosimulators_utils/config.py | 139 +++++++++++++++++++--------- 2 files changed, 105 insertions(+), 49 deletions(-) diff --git a/biosimulators_utils/combine/exec.py b/biosimulators_utils/combine/exec.py index ba4742ca..ec6e278e 100644 --- a/biosimulators_utils/combine/exec.py +++ b/biosimulators_utils/combine/exec.py @@ -6,6 +6,13 @@ :License: MIT """ +import copy +import datetime +import glob +import os +import tempfile +import shutil +import types # noqa: F401 from ..archive.io import ArchiveWriter from ..archive.utils import build_archive_from_paths from ..config import get_config, Config # noqa: F401 @@ -22,13 +29,7 @@ from .utils import get_sedml_contents, get_summary_sedml_contents from .validation import validate from ..viz.data_model import VizFormat # noqa: F401 -import copy -import datetime -import glob -import os -import tempfile -import shutil -import types # noqa: F401 + __all__ = [ 'exec_sedml_docs_in_archive', diff --git a/biosimulators_utils/config.py b/biosimulators_utils/config.py index 782fb105..85644aa0 100644 --- a/biosimulators_utils/config.py +++ b/biosimulators_utils/config.py @@ -13,19 +13,21 @@ from datetime import datetime from typing import Dict, List, Union from kisao import AlgorithmSubstitutionPolicy # noqa: F401 +import appdirs from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel -from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, \ - OmexMetadataSchema +from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, \ + OmexMetadataOutputFormat, OmexMetadataSchema from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 from biosimulators_utils.viz.data_model import VizFormat # noqa: F401 -import appdirs + __all__ = [ 'Config', 'get_config', 'Colors', - 'get_app_dirs', 'acceptable_report_formats', - 'acceptable_viz_formats' + 'get_app_dirs', 'get_acceptable_report_formats', + 'get_acceptable_viz_formats' ] + CURRENT_PLATFORM = platform.system() try: assert CURRENT_PLATFORM == "Darwin" @@ -48,7 +50,7 @@ class Config(object): """ Configuration - + Attributes: OMEX_METADATA_INPUT_FORMAT (:obj:`OmexMetadataInputFormat`): format to validate OMEX Metadata files against OMEX_METADATA_OUTPUT_FORMAT (:obj:`OmexMetadataOutputFormat`): format to export OMEX Metadata files @@ -82,8 +84,8 @@ class Config(object): VERBOSE (:obj:`bool`): whether to display the detailed output of the execution of each task DEBUG (:obj:`bool`): whether to raise exceptions rather than capturing them STDOUT_LEVEL (:obj:`StandardOutputErrorCapturerLevel`): level at which to log the output + CUSTOM_SETTINGS (:obj:`**kwargs`) """ - def __init__(self, OMEX_METADATA_INPUT_FORMAT=DEFAULT_OMEX_METADATA_INPUT_FORMAT, OMEX_METADATA_OUTPUT_FORMAT=DEFAULT_OMEX_METADATA_OUTPUT_FORMAT, @@ -153,7 +155,8 @@ def __init__(self, DEBUG (:obj:`bool`, optional): whether to raise exceptions rather than capturing them STDOUT_LEVEL (:obj:`StandardOutputErrorCapturerLevel`): level at which to log the output """ - self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT + + self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT # noqa C0103 self.OMEX_METADATA_OUTPUT_FORMAT = OMEX_METADATA_OUTPUT_FORMAT self.OMEX_METADATA_SCHEMA = OMEX_METADATA_SCHEMA self.VALIDATE_OMEX_MANIFESTS = VALIDATE_OMEX_MANIFESTS @@ -167,8 +170,8 @@ def __init__(self, self.COLLECT_COMBINE_ARCHIVE_RESULTS = COLLECT_COMBINE_ARCHIVE_RESULTS self.COLLECT_SED_DOCUMENT_RESULTS = COLLECT_SED_DOCUMENT_RESULTS self.SAVE_PLOT_DATA = SAVE_PLOT_DATA - self.REPORT_FORMATS = REPORT_FORMATS or [ReportFormat.csv] - self.VIZ_FORMATS = VIZ_FORMATS or [VizFormat.pdf] + self.REPORT_FORMATS = [ReportFormat.csv] if not REPORT_FORMATS else REPORT_FORMATS + self.VIZ_FORMATS = [VizFormat.pdf] if not VIZ_FORMATS else VIZ_FORMATS self.H5_REPORTS_PATH = H5_REPORTS_PATH self.REPORTS_PATH = REPORTS_PATH self.PLOTS_PATH = PLOTS_PATH @@ -184,7 +187,7 @@ def __init__(self, self.DEBUG = DEBUG self.STDOUT_LEVEL = STDOUT_LEVEL self.CUSTOM_SETTINGS = self.__getcustomsettings(CUSTOM_SETTINGS) - if "EASY_LOG" in self.CUSTOM_SETTINGS: + if "EASY_LOG" in CUSTOM_SETTINGS: self.logger = self.easy_log() def __getcustomsettings(self, settings: Dict = None): @@ -198,9 +201,9 @@ def easy_log(self): def get_config(report_format: str = None, viz_format: str = None, + easy_log: bool = False, acceptable_report_formats: Union[List[str], ReportFormat] = ReportFormat, acceptable_viz_formats: Union[List[str], VizFormat] = VizFormat, - easy_log: bool = False, *_default_format_settings) -> Config: """ Get the configuration based on specified optional settings. Handles sets default values for `report_format` and `viz_format` if these respective variables are empty. @@ -217,17 +220,17 @@ def get_config(report_format: str = None, Returns: :obj:`Config`: configuration """ - + if not _default_format_settings: # get _default_format_settings = ('csv', 'pdf') # set - user_report_format = verify_formats( + user_report_format = _verify_formats( report_format, acceptable_report_formats, _default_format_settings[0] ) - user_viz_format = verify_formats( + user_viz_format = _verify_formats( viz_format, acceptable_viz_formats, _default_format_settings[1] @@ -237,7 +240,7 @@ def get_config(report_format: str = None, if report_formats: report_formats = [ - ReportFormat(format.strip().lower()) for format in report_formats.split(',') + ReportFormat(f.strip().lower()) for f in report_formats.split(',') ] else: report_formats = [] @@ -245,7 +248,7 @@ def get_config(report_format: str = None, viz_formats = os.environ.get('VIZ_FORMATS', user_viz_format).strip() if viz_formats: viz_formats = [ - VizFormat(format.strip().lower()) for format in viz_formats.split(',') + VizFormat(f.strip().lower()) for f in viz_formats.split(',') ] else: viz_formats = [] @@ -257,17 +260,22 @@ def get_config(report_format: str = None, 'OMEX_METADATA_OUTPUT_FORMAT', DEFAULT_OMEX_METADATA_OUTPUT_FORMAT)), OMEX_METADATA_SCHEMA=OmexMetadataSchema(os.environ.get( 'OMEX_METADATA_SCHEMA', DEFAULT_OMEX_METADATA_SCHEMA)), - VALIDATE_OMEX_MANIFESTS=os.environ.get('VALIDATE_OMEX_MANIFESTS', '1').lower() in ['1', 'true'], + VALIDATE_OMEX_MANIFESTS=os.environ.get( + 'VALIDATE_OMEX_MANIFESTS', '1').lower() in ['1', 'true'], VALIDATE_SEDML=os.environ.get('VALIDATE_SEDML', '1').lower() in ['1', 'true'], VALIDATE_SEDML_MODELS=os.environ.get('VALIDATE_SEDML_MODELS', '1').lower() in ['1', 'true'], - VALIDATE_IMPORTED_MODEL_FILES=os.environ.get('VALIDATE_IMPORTED_MODEL_FILES', '1').lower() in ['1', 'true'], - VALIDATE_OMEX_METADATA=os.environ.get('VALIDATE_OMEX_METADATA', '1').lower() in ['1', 'true'], + VALIDATE_IMPORTED_MODEL_FILES=os.environ.get( + 'VALIDATE_IMPORTED_MODEL_FILES', '1').lower() in ['1', 'true'], + VALIDATE_OMEX_METADATA=os.environ.get( + 'VALIDATE_OMEX_METADATA', '1').lower() in ['1', 'true'], VALIDATE_IMAGES=os.environ.get('VALIDATE_IMAGES', '1').lower() in ['1', 'true'], VALIDATE_RESULTS=os.environ.get('VALIDATE_RESULTS', '1').lower() in ['1', 'true'], ALGORITHM_SUBSTITUTION_POLICY=AlgorithmSubstitutionPolicy(os.environ.get( 'ALGORITHM_SUBSTITUTION_POLICY', DEFAULT_ALGORITHM_SUBSTITUTION_POLICY)), - COLLECT_COMBINE_ARCHIVE_RESULTS=os.environ.get('COLLECT_COMBINE_ARCHIVE_RESULTS', '0').lower() in ['1', 'true'], - COLLECT_SED_DOCUMENT_RESULTS=os.environ.get('COLLECT_SED_DOCUMENT_RESULTS', '0').lower() in ['1', 'true'], + COLLECT_COMBINE_ARCHIVE_RESULTS=os.environ.get( + 'COLLECT_COMBINE_ARCHIVE_RESULTS', '0').lower() in ['1', 'true'], + COLLECT_SED_DOCUMENT_RESULTS=os.environ.get( + 'COLLECT_SED_DOCUMENT_RESULTS', '0').lower() in ['1', 'true'], SAVE_PLOT_DATA=os.environ.get('SAVE_PLOT_DATA', '1').lower() in ['1', 'true'], REPORT_FORMATS=report_formats, VIZ_FORMATS=viz_formats, @@ -275,14 +283,18 @@ def get_config(report_format: str = None, REPORTS_PATH=os.environ.get('REPORTS_PATH', DEFAULT_REPORTS_PATH), PLOTS_PATH=os.environ.get('PLOTS_PATH', DEFAULT_PLOTS_PATH), BUNDLE_OUTPUTS=os.environ.get('BUNDLE_OUTPUTS', '1').lower() in ['1', 'true'], - KEEP_INDIVIDUAL_OUTPUTS=os.environ.get('KEEP_INDIVIDUAL_OUTPUTS', '1').lower() in ['1', 'true'], + KEEP_INDIVIDUAL_OUTPUTS=os.environ.get( + 'KEEP_INDIVIDUAL_OUTPUTS', '1').lower() in ['1', 'true'], LOG=os.environ.get('LOG', '1').lower() in ['1', 'true'], LOG_PATH=os.environ.get('LOG_PATH', DEFAULT_LOG_PATH), - BIOSIMULATORS_API_ENDPOINT=os.environ.get('BIOSIMULATORS_API_ENDPOINT', DEFAULT_BIOSIMULATORS_API_ENDPOINT), - BIOSIMULATIONS_API_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_ENDPOINT), + BIOSIMULATORS_API_ENDPOINT=os.environ.get( + 'BIOSIMULATORS_API_ENDPOINT', DEFAULT_BIOSIMULATORS_API_ENDPOINT), + BIOSIMULATIONS_API_ENDPOINT=os.environ.get( + 'BIOSIMULATIONS_API_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_ENDPOINT), BIOSIMULATIONS_API_AUTH_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_AUTH_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT), - BIOSIMULATIONS_API_AUDIENCE=os.environ.get('BIOSIMULATIONS_API_AUDIENCE', DEFAULT_BIOSIMULATIONS_API_AUDIENCE), + BIOSIMULATIONS_API_AUDIENCE=os.environ.get( + 'BIOSIMULATIONS_API_AUDIENCE', DEFAULT_BIOSIMULATIONS_API_AUDIENCE), VERBOSE=os.environ.get('VERBOSE', '1').lower() in ['1', 'true'], DEBUG=os.environ.get('DEBUG', '0').lower() in ['1', 'true'], STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL), @@ -309,7 +321,8 @@ def get_config(report_format: str = None, def get_app_dirs(): - """ Get the directories for the application + """ + Get the directories for the application Returns: :obj:`appdirs.AppDirs`: application directories @@ -317,25 +330,50 @@ def get_app_dirs(): return appdirs.AppDirs("BioSimulatorsUtils", "BioSimulatorsTeam") -def acceptable_viz_formats(): +def get_acceptable_viz_formats() -> List[str]: + """ + Return a list of all the acceptable visualization formats. + + Wrapper for `get_acceptable_formats`. + + Returns: + :func:`get_acceptable_formats()`: list of acceptable viz formats + """ return get_acceptable_formats(VizFormat) -def acceptable_report_formats(): +def get_acceptable_report_formats() -> List[str]: + """ + Return a list of all the acceptable report formats. + + Wrapper for `get_acceptable_formats`. + + Returns: + :func:`get_acceptable_formats()`: list of acceptable report formats + """ return get_acceptable_formats(ReportFormat) -def get_acceptable_formats(acceptable_formats: enum.Enum): +def get_acceptable_formats(acceptable_formats: enum.Enum) -> List[str]: + """ + Return a list of all the acceptable formats for a given format type. + + Args: + :enum.Enum:`acceptable_formats`: instance of either `ReportFormat` or `VizFormat`. + + Returns: + :list: list of acceptable formats for the given `acceptable_format` + """ return [v.value for v in acceptable_formats] -def verify_formats(format_type: str, acceptable_format: enum.Enum, default: str): +def _verify_formats(format_type: str, acceptable_format: enum.Enum, default: str): def verify_format(form_type, acceptable_form): acceptable_formats = get_acceptable_formats(acceptable_form) if form_type not in acceptable_formats: print( f'''Sorry, you must enter one of the following acceptable formats: - {acceptable_formats}. \nSetting to default format: {default}''' + {acceptable_formats}.\nSetting to default format: {default}''' ) return False else: @@ -346,11 +384,27 @@ def verify_format(form_type, acceptable_form): class EasyLog: + """Utility class for quickly logging and capturing executed actions. + + Attributes: + working_file: (:obj:`str`): currently executed file. Defaults to `__file__` + + log: (:obj:`dict`): collection of all logs. + + index: (:obj:`list`): a list of indexes that directly correspond to the cardinality of the log. + """ + def __init__(self, log_dir, fresh: bool = False): + """ + Args: + log_dir: (:obj:`str`): filepath of logging dir. + + fresh: (:obj:`bool`): If `True`, destroy previous log, logdir and creates a new one. + """ self.working_file = __file__ self._make_logdir(log_dir, fresh) self.log = {} - self.index = list(range(len(self.log))) + self._index = list(range(self.__getsize__())) def __getsize__(self): return len(self.log) @@ -359,7 +413,8 @@ def __getnow__(self): return datetime.now().strftime("%d.%m.%Y..%H.%M.%S") def _make_logdir(self, log_dir: str, fresh_log: bool): - make_dir = lambda: os.os.mkdir(log_dir) if not os.path.exists(log_dir) else None + def make_dir(): + return os.mkdir(log_dir) if not os.path.exists(log_dir) else None if fresh_log: filepaths = [] for root, _, files in os.walk(log_dir): @@ -371,20 +426,20 @@ def _make_logdir(self, log_dir: str, fresh_log: bool): else: make_dir() - def add_msg(self, message, function="none", status="none"): + def add_msg(self, message, func="none", status="none"): size = self.__getsize__() entry_number = 1 if size < 1 else size now = self.__getnow__() - verify = lambda v: v != "none" - function = function.__name__ if verify(function) else function + def verify(v): return v != "none" + func = func.__name__ if verify(func) else func status = str(status) entry = f"""{now} | NOTES: {message} | CALLED FROM: {self.working_file} - | METHOD CALLED: {function} | STATUS: {status.upper()}""" + | METHOD CALLED: {func} | STATUS: {status.upper()}""" self.log[entry_number] = entry return self.log def flush(self): - for n in self.index: + for n in self._index: self.log.pop(n) return self.log @@ -392,10 +447,10 @@ def write(self, log_fp: str = None): if not log_fp: now = self.__getnow__() log_fp = f"log_{now}.json" - with open(log_fp, "w"): - json.dump(self.log, log_fp, indent=4) + with open(log_fp, "w") as f: + json.dump(self.log, f, indent=4) def flush_log(self, log_fp: str = None): self.write(log_fp) self.flush() - return + From ad7843bb50d87722b7645236c6cd7823a17579db Mon Sep 17 00:00:00 2001 From: "A.P" <105522603+AlexPatrie@users.noreply.github.com> Date: Thu, 11 May 2023 04:12:02 +0000 Subject: [PATCH 13/16] minor changes --- biosimulators_utils/config.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/biosimulators_utils/config.py b/biosimulators_utils/config.py index 85644aa0..e130c596 100644 --- a/biosimulators_utils/config.py +++ b/biosimulators_utils/config.py @@ -29,11 +29,8 @@ CURRENT_PLATFORM = platform.system() -try: - assert CURRENT_PLATFORM == "Darwin" - DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.python -except AssertionError as e: - DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c +DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c if not "Darwin" in CURRENT_PLATFORM \ + else StandardOutputErrorCapturerLevel.python DEFAULT_OMEX_METADATA_INPUT_FORMAT = OmexMetadataInputFormat.rdfxml DEFAULT_OMEX_METADATA_OUTPUT_FORMAT = OmexMetadataOutputFormat.rdfxml_abbrev DEFAULT_OMEX_METADATA_SCHEMA = OmexMetadataSchema.biosimulations @@ -50,7 +47,7 @@ class Config(object): """ Configuration - + Attributes: OMEX_METADATA_INPUT_FORMAT (:obj:`OmexMetadataInputFormat`): format to validate OMEX Metadata files against OMEX_METADATA_OUTPUT_FORMAT (:obj:`OmexMetadataOutputFormat`): format to export OMEX Metadata files @@ -86,6 +83,7 @@ class Config(object): STDOUT_LEVEL (:obj:`StandardOutputErrorCapturerLevel`): level at which to log the output CUSTOM_SETTINGS (:obj:`**kwargs`) """ + def __init__(self, OMEX_METADATA_INPUT_FORMAT=DEFAULT_OMEX_METADATA_INPUT_FORMAT, OMEX_METADATA_OUTPUT_FORMAT=DEFAULT_OMEX_METADATA_OUTPUT_FORMAT, @@ -156,7 +154,7 @@ def __init__(self, STDOUT_LEVEL (:obj:`StandardOutputErrorCapturerLevel`): level at which to log the output """ - self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT # noqa C0103 + self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT # noqa C0103 self.OMEX_METADATA_OUTPUT_FORMAT = OMEX_METADATA_OUTPUT_FORMAT self.OMEX_METADATA_SCHEMA = OMEX_METADATA_SCHEMA self.VALIDATE_OMEX_MANIFESTS = VALIDATE_OMEX_MANIFESTS @@ -207,20 +205,20 @@ def get_config(report_format: str = None, *_default_format_settings) -> Config: """ Get the configuration based on specified optional settings. Handles sets default values for `report_format` and `viz_format` if these respective variables are empty. - + Args: :str:`report_format`: output format for reports. Defaults to `None` - + :str:`viz_format`: output format for visualizations. Defaults to `None` - + :Union:`acceptable_report_formats`: acceptable formats for output report data. Defaults to `biosimulators_utils.report.data_model.ReportFormat` - + :Union:`acceptable_viz_formats`: acceptable formats for output viz data. Defaults to `biosimulators_utils.viz.data_model.VizFormat` - + Returns: :obj:`Config`: configuration """ - + if not _default_format_settings: # get _default_format_settings = ('csv', 'pdf') # set @@ -385,7 +383,7 @@ def verify_format(form_type, acceptable_form): class EasyLog: """Utility class for quickly logging and capturing executed actions. - + Attributes: working_file: (:obj:`str`): currently executed file. Defaults to `__file__` @@ -398,7 +396,7 @@ def __init__(self, log_dir, fresh: bool = False): """ Args: log_dir: (:obj:`str`): filepath of logging dir. - + fresh: (:obj:`bool`): If `True`, destroy previous log, logdir and creates a new one. """ self.working_file = __file__ @@ -453,4 +451,3 @@ def write(self, log_fp: str = None): def flush_log(self, log_fp: str = None): self.write(log_fp) self.flush() - From daa48ee44032062daa1a6571387075be6cfcd1d7 Mon Sep 17 00:00:00 2001 From: "A.P" <105522603+AlexPatrie@users.noreply.github.com> Date: Thu, 11 May 2023 04:16:01 +0000 Subject: [PATCH 14/16] implementation of commit --- biosimulators_utils/combine/exec.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/biosimulators_utils/combine/exec.py b/biosimulators_utils/combine/exec.py index ec6e278e..4e5737a0 100644 --- a/biosimulators_utils/combine/exec.py +++ b/biosimulators_utils/combine/exec.py @@ -39,8 +39,8 @@ def exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, sed_doc_executer_supported_features=(Task, Report, DataSet, Plot2D, Curve, Plot3D, Surface), sed_doc_executer_logged_features=(Task, Report, DataSet, Plot2D, Curve, Plot3D, Surface), - log_level=StandardOutputErrorCapturerLevel.c, - config=None): + log_level: StandardOutputErrorCapturerLevel = None, + config: Config = None): """ Execute the SED-ML files in a COMBINE/OMEX archive (execute tasks and save outputs) Args: @@ -95,7 +95,10 @@ def sed_doc_executer(doc, working_dir, base_out_path, rel_out_path=None, * :obj:`CombineArchiveLog`: log """ if not config: - config = get_config() + config = get_config(easy_log=True) + + if not log_level: + log_level = config.STDOUT_LEVEL with StandardOutputErrorCapturer(relay=True, level=log_level, disabled=not config.LOG) as archive_captured: verbose = config.VERBOSE From 99e36eeb10d0b032c83db17749f08d03e05ae8e9 Mon Sep 17 00:00:00 2001 From: "A.P" <105522603+AlexPatrie@users.noreply.github.com> Date: Thu, 11 May 2023 11:48:16 -0400 Subject: [PATCH 15/16] Delete wip_config.py --- biosimulators_utils/wip_config.py | 306 ------------------------------ 1 file changed, 306 deletions(-) delete mode 100644 biosimulators_utils/wip_config.py diff --git a/biosimulators_utils/wip_config.py b/biosimulators_utils/wip_config.py deleted file mode 100644 index 9aa81f7f..00000000 --- a/biosimulators_utils/wip_config.py +++ /dev/null @@ -1,306 +0,0 @@ -import enum -import os -import json -import platform -from datetime import datetime -from typing import Dict, List, Union -from kisao import AlgorithmSubstitutionPolicy # noqa: F401 -from biosimulators_utils.log.data_model import StandardOutputErrorCapturerLevel -from biosimulators_utils.omex_meta.data_model import OmexMetadataInputFormat, OmexMetadataOutputFormat, \ - OmexMetadataSchema -from biosimulators_utils.report.data_model import ReportFormat # noqa: F401 -from biosimulators_utils.viz.data_model import VizFormat # noqa: F401 -import appdirs - -__all__ = [ - 'Config', 'get_config', 'Colors', - 'get_app_dirs', 'acceptable_report_formats', - 'acceptable_viz_formats' -] - - -CURRENT_PLATFORM = platform.system() -try: - assert CURRENT_PLATFORM == "Darwin" - DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.python -except AssertionError as e: - DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c -DEFAULT_OMEX_METADATA_INPUT_FORMAT = OmexMetadataInputFormat.rdfxml -DEFAULT_OMEX_METADATA_OUTPUT_FORMAT = OmexMetadataOutputFormat.rdfxml_abbrev -DEFAULT_OMEX_METADATA_SCHEMA = OmexMetadataSchema.biosimulations -DEFAULT_ALGORITHM_SUBSTITUTION_POLICY = AlgorithmSubstitutionPolicy.SIMILAR_VARIABLES -DEFAULT_H5_REPORTS_PATH = 'reports.h5' -DEFAULT_REPORTS_PATH = 'reports.zip' -DEFAULT_PLOTS_PATH = 'plots.zip' -DEFAULT_LOG_PATH = 'log.yml' -DEFAULT_BIOSIMULATORS_API_ENDPOINT = 'https://api.biosimulators.org/' -DEFAULT_BIOSIMULATIONS_API_ENDPOINT = 'https://api.biosimulations.org/' -DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT = 'https://auth.biosimulations.org/oauth/token' -DEFAULT_BIOSIMULATIONS_API_AUDIENCE = 'api.biosimulations.org' - - -class Config(object): - def __init__(self, - OMEX_METADATA_INPUT_FORMAT=DEFAULT_OMEX_METADATA_INPUT_FORMAT, - OMEX_METADATA_OUTPUT_FORMAT=DEFAULT_OMEX_METADATA_OUTPUT_FORMAT, - OMEX_METADATA_SCHEMA=DEFAULT_OMEX_METADATA_SCHEMA, - VALIDATE_OMEX_MANIFESTS=True, - VALIDATE_SEDML=True, - VALIDATE_SEDML_MODELS=True, - VALIDATE_IMPORTED_MODEL_FILES=True, - VALIDATE_OMEX_METADATA=True, - VALIDATE_IMAGES=True, - VALIDATE_RESULTS=True, - ALGORITHM_SUBSTITUTION_POLICY=DEFAULT_ALGORITHM_SUBSTITUTION_POLICY, - COLLECT_COMBINE_ARCHIVE_RESULTS=False, - COLLECT_SED_DOCUMENT_RESULTS=False, - SAVE_PLOT_DATA=True, - REPORT_FORMATS=None, - VIZ_FORMATS=None, - H5_REPORTS_PATH=DEFAULT_H5_REPORTS_PATH, - REPORTS_PATH=DEFAULT_REPORTS_PATH, - PLOTS_PATH=DEFAULT_PLOTS_PATH, - BUNDLE_OUTPUTS=True, - KEEP_INDIVIDUAL_OUTPUTS=True, - LOG=True, - LOG_PATH=DEFAULT_LOG_PATH, - BIOSIMULATORS_API_ENDPOINT=DEFAULT_BIOSIMULATORS_API_ENDPOINT, - BIOSIMULATIONS_API_ENDPOINT=DEFAULT_BIOSIMULATIONS_API_ENDPOINT, - BIOSIMULATIONS_API_AUTH_ENDPOINT=DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT, - BIOSIMULATIONS_API_AUDIENCE=DEFAULT_BIOSIMULATIONS_API_AUDIENCE, - VERBOSE=False, - DEBUG=False, - STDOUT_LEVEL=DEFAULT_STDOUT_LEVEL, - **CUSTOM_SETTINGS): - - self.OMEX_METADATA_INPUT_FORMAT = OMEX_METADATA_INPUT_FORMAT - self.OMEX_METADATA_OUTPUT_FORMAT = OMEX_METADATA_OUTPUT_FORMAT - self.OMEX_METADATA_SCHEMA = OMEX_METADATA_SCHEMA - self.VALIDATE_OMEX_MANIFESTS = VALIDATE_OMEX_MANIFESTS - self.VALIDATE_SEDML = VALIDATE_SEDML - self.VALIDATE_SEDML_MODELS = VALIDATE_SEDML_MODELS - self.VALIDATE_IMPORTED_MODEL_FILES = VALIDATE_IMPORTED_MODEL_FILES - self.VALIDATE_OMEX_METADATA = VALIDATE_OMEX_METADATA - self.VALIDATE_IMAGES = VALIDATE_IMAGES - self.VALIDATE_RESULTS = VALIDATE_RESULTS - self.ALGORITHM_SUBSTITUTION_POLICY = ALGORITHM_SUBSTITUTION_POLICY - self.COLLECT_COMBINE_ARCHIVE_RESULTS = COLLECT_COMBINE_ARCHIVE_RESULTS - self.COLLECT_SED_DOCUMENT_RESULTS = COLLECT_SED_DOCUMENT_RESULTS - self.SAVE_PLOT_DATA = SAVE_PLOT_DATA - self.REPORT_FORMATS = [ReportFormat.csv] if not REPORT_FORMATS else REPORT_FORMATS - self.VIZ_FORMATS = [VizFormat.pdf] if not VIZ_FORMATS else VIZ_FORMATS - self.H5_REPORTS_PATH = H5_REPORTS_PATH - self.REPORTS_PATH = REPORTS_PATH - self.PLOTS_PATH = PLOTS_PATH - self.BUNDLE_OUTPUTS = BUNDLE_OUTPUTS - self.KEEP_INDIVIDUAL_OUTPUTS = KEEP_INDIVIDUAL_OUTPUTS - self.LOG = LOG - self.LOG_PATH = LOG_PATH - self.BIOSIMULATORS_API_ENDPOINT = BIOSIMULATORS_API_ENDPOINT - self.BIOSIMULATIONS_API_ENDPOINT = BIOSIMULATIONS_API_ENDPOINT - self.BIOSIMULATIONS_API_AUTH_ENDPOINT = BIOSIMULATIONS_API_AUTH_ENDPOINT - self.BIOSIMULATIONS_API_AUDIENCE = BIOSIMULATIONS_API_AUDIENCE - self.VERBOSE = VERBOSE - self.DEBUG = DEBUG - self.STDOUT_LEVEL = STDOUT_LEVEL - self.CUSTOM_SETTINGS = self.__getcustomsettings(CUSTOM_SETTINGS) - if "EASY_LOG" in CUSTOM_SETTINGS: - self.logger = self.easy_log() - - def __getcustomsettings(self, settings: Dict = None): - for key in settings.keys(): - setattr(self, key, settings[key]) - return self - - def easy_log(self): - return EasyLog(os.getcwd()) - - -def get_config(report_format: str = None, - viz_format: str = None, - acceptable_report_formats: Union[List[str], ReportFormat] = ReportFormat, - acceptable_viz_formats: Union[List[str], VizFormat] = VizFormat, - easy_log: bool = False, - *_default_format_settings) -> Config: - if not _default_format_settings: # get - _default_format_settings = ('csv', 'pdf') # set - - user_report_format = verify_formats( - report_format, - acceptable_report_formats, - _default_format_settings[0] - ) - - user_viz_format = verify_formats( - viz_format, - acceptable_viz_formats, - _default_format_settings[1] - ) - - print(user_viz_format, user_report_format) - - report_formats = os.environ.get('REPORT_FORMATS', user_report_format).strip() - - if report_formats: - report_formats = [ - ReportFormat(f.strip().lower()) for f in report_formats.split(',') - ] - else: - report_formats = [] - - viz_formats = os.environ.get('VIZ_FORMATS', user_viz_format).strip() - if viz_formats: - viz_formats = [ - VizFormat(f.strip().lower()) for f in viz_formats.split(',') - ] - else: - viz_formats = [] - - return Config( - OMEX_METADATA_INPUT_FORMAT=OmexMetadataInputFormat(os.environ.get( - 'OMEX_METADATA_INPUT_FORMAT', DEFAULT_OMEX_METADATA_INPUT_FORMAT)), - OMEX_METADATA_OUTPUT_FORMAT=OmexMetadataOutputFormat(os.environ.get( - 'OMEX_METADATA_OUTPUT_FORMAT', DEFAULT_OMEX_METADATA_OUTPUT_FORMAT)), - OMEX_METADATA_SCHEMA=OmexMetadataSchema(os.environ.get( - 'OMEX_METADATA_SCHEMA', DEFAULT_OMEX_METADATA_SCHEMA)), - VALIDATE_OMEX_MANIFESTS=os.environ.get('VALIDATE_OMEX_MANIFESTS', '1').lower() in ['1', 'true'], - VALIDATE_SEDML=os.environ.get('VALIDATE_SEDML', '1').lower() in ['1', 'true'], - VALIDATE_SEDML_MODELS=os.environ.get('VALIDATE_SEDML_MODELS', '1').lower() in ['1', 'true'], - VALIDATE_IMPORTED_MODEL_FILES=os.environ.get('VALIDATE_IMPORTED_MODEL_FILES', '1').lower() in ['1', 'true'], - VALIDATE_OMEX_METADATA=os.environ.get('VALIDATE_OMEX_METADATA', '1').lower() in ['1', 'true'], - VALIDATE_IMAGES=os.environ.get('VALIDATE_IMAGES', '1').lower() in ['1', 'true'], - VALIDATE_RESULTS=os.environ.get('VALIDATE_RESULTS', '1').lower() in ['1', 'true'], - ALGORITHM_SUBSTITUTION_POLICY=AlgorithmSubstitutionPolicy(os.environ.get( - 'ALGORITHM_SUBSTITUTION_POLICY', DEFAULT_ALGORITHM_SUBSTITUTION_POLICY)), - COLLECT_COMBINE_ARCHIVE_RESULTS=os.environ.get('COLLECT_COMBINE_ARCHIVE_RESULTS', '0').lower() in ['1', 'true'], - COLLECT_SED_DOCUMENT_RESULTS=os.environ.get('COLLECT_SED_DOCUMENT_RESULTS', '0').lower() in ['1', 'true'], - SAVE_PLOT_DATA=os.environ.get('SAVE_PLOT_DATA', '1').lower() in ['1', 'true'], - REPORT_FORMATS=report_formats, - VIZ_FORMATS=viz_formats, - H5_REPORTS_PATH=os.environ.get('H5_REPORTS_PATH', DEFAULT_H5_REPORTS_PATH), - REPORTS_PATH=os.environ.get('REPORTS_PATH', DEFAULT_REPORTS_PATH), - PLOTS_PATH=os.environ.get('PLOTS_PATH', DEFAULT_PLOTS_PATH), - BUNDLE_OUTPUTS=os.environ.get('BUNDLE_OUTPUTS', '1').lower() in ['1', 'true'], - KEEP_INDIVIDUAL_OUTPUTS=os.environ.get('KEEP_INDIVIDUAL_OUTPUTS', '1').lower() in ['1', 'true'], - LOG=os.environ.get('LOG', '1').lower() in ['1', 'true'], - LOG_PATH=os.environ.get('LOG_PATH', DEFAULT_LOG_PATH), - BIOSIMULATORS_API_ENDPOINT=os.environ.get('BIOSIMULATORS_API_ENDPOINT', DEFAULT_BIOSIMULATORS_API_ENDPOINT), - BIOSIMULATIONS_API_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_ENDPOINT', DEFAULT_BIOSIMULATIONS_API_ENDPOINT), - BIOSIMULATIONS_API_AUTH_ENDPOINT=os.environ.get('BIOSIMULATIONS_API_AUTH_ENDPOINT', - DEFAULT_BIOSIMULATIONS_API_AUTH_ENDPOINT), - BIOSIMULATIONS_API_AUDIENCE=os.environ.get('BIOSIMULATIONS_API_AUDIENCE', DEFAULT_BIOSIMULATIONS_API_AUDIENCE), - VERBOSE=os.environ.get('VERBOSE', '1').lower() in ['1', 'true'], - DEBUG=os.environ.get('DEBUG', '0').lower() in ['1', 'true'], - STDOUT_LEVEL=os.environ.get('STDOUT_LEVEL', DEFAULT_STDOUT_LEVEL), - EASY_LOG=None if not easy_log else easy_log - ) - - -Colors = enum.Enum('Colors', - { - 'queued': 'cyan', - 'success': 'blue', - 'succeeded': 'blue', - 'running': 'green', - 'pass': 'green', - 'passed': 'green', - 'failure': 'red', - 'failed': 'red', - 'skip': 'magenta', - 'skipped': 'magenta', - 'warning': 'yellow', - 'warned': 'yellow', - }, - type=str) - - -def get_app_dirs(): - """ Get the directories for the application - - Returns: - :obj:`appdirs.AppDirs`: application directories - """ - return appdirs.AppDirs("BioSimulatorsUtils", "BioSimulatorsTeam") - - -def acceptable_viz_formats(): - return get_acceptable_formats(VizFormat) - - -def acceptable_report_formats(): - return get_acceptable_formats(ReportFormat) - - -def get_acceptable_formats(acceptable_formats: enum.Enum): - return [v.value for v in acceptable_formats] - - -def verify_formats(format_type: str, acceptable_format: enum.Enum, default: str): - def verify_format(form_type, acceptable_form): - acceptable_formats = get_acceptable_formats(acceptable_form) - if form_type not in acceptable_formats: - print( - f'''Sorry, you must enter one of the following acceptable formats: - {acceptable_formats}.\nSetting to default format: {default}''' - ) - return False - else: - return True - - return default if not verify_format(format_type, acceptable_format) \ - else format_type - - -class EasyLog: - def __init__(self, log_dir, fresh: bool = False): - self.working_file = __file__ - self._make_logdir(log_dir, fresh) - self.log = {} - self.index = list(range(len(self.log))) - - def __getsize__(self): - return len(self.log) - - def __getnow__(self): - return datetime.now().strftime("%d.%m.%Y..%H.%M.%S") - - def _make_logdir(self, log_dir: str, fresh_log: bool): - make_dir = lambda: os.os.mkdir(log_dir) if not os.path.exists(log_dir) else None - if fresh_log: - filepaths = [] - for root, _, files in os.walk(log_dir): - filepaths.append(os.path.join(root, f) for f in files) - for f in filepaths: - os.remove(f) - os.rmdir(log_dir) - make_dir() - else: - make_dir() - - def add_msg(self, message, func="none", status="none"): - size = self.__getsize__() - entry_number = 1 if size < 1 else size - now = self.__getnow__() - verify = lambda v: v != "none" - func = func.__name__ if verify(func) else func - status = str(status) - entry = f"""{now} | NOTES: {message} | CALLED FROM: {self.working_file} - | METHOD CALLED: {func} | STATUS: {status.upper()}""" - self.log[entry_number] = entry - return self.log - - def flush(self): - for n in self.index: - self.log.pop(n) - return self.log - - def write(self, log_fp: str = None): - if not log_fp: - now = self.__getnow__() - log_fp = f"log_{now}.json" - with open(log_fp, "w"): - json.dump(self.log, log_fp, indent=4) - - def flush_log(self, log_fp: str = None): - self.write(log_fp) - self.flush() From ab20f1a442f79664f9c35bfbf489bb596a83bfc2 Mon Sep 17 00:00:00 2001 From: alex patrie Date: Thu, 11 May 2023 15:37:59 -0400 Subject: [PATCH 16/16] linting adjustments --- biosimulators_utils/config.py | 4 ++-- biosimulators_utils/sedml/exec.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/biosimulators_utils/config.py b/biosimulators_utils/config.py index e130c596..f61c1d0c 100644 --- a/biosimulators_utils/config.py +++ b/biosimulators_utils/config.py @@ -24,12 +24,12 @@ __all__ = [ 'Config', 'get_config', 'Colors', 'get_app_dirs', 'get_acceptable_report_formats', - 'get_acceptable_viz_formats' + 'get_acceptable_viz_formats', 'EasyLog' ] CURRENT_PLATFORM = platform.system() -DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c if not "Darwin" in CURRENT_PLATFORM \ +DEFAULT_STDOUT_LEVEL = StandardOutputErrorCapturerLevel.c if "Darwin" not in CURRENT_PLATFORM \ else StandardOutputErrorCapturerLevel.python DEFAULT_OMEX_METADATA_INPUT_FORMAT = OmexMetadataInputFormat.rdfxml DEFAULT_OMEX_METADATA_OUTPUT_FORMAT = OmexMetadataOutputFormat.rdfxml_abbrev diff --git a/biosimulators_utils/sedml/exec.py b/biosimulators_utils/sedml/exec.py index 05696409..170e624a 100644 --- a/biosimulators_utils/sedml/exec.py +++ b/biosimulators_utils/sedml/exec.py @@ -105,19 +105,19 @@ def exec_task(task, variables, preprocessed_task=None, log=None, config=None, ** * :obj:`ReportResults`: results of each report * :obj:`SedDocumentLog`: log of the document """ - if not config: + if not config: # python:S3776 config = get_config(easy_log=True) - if not log_level: + if not log_level: # python:S3776 log_level = config.STDOUT_LEVEL # process arguments - if not isinstance(doc, SedDocument): + if not isinstance(doc, SedDocument): # python:S3776 doc = SedmlSimulationReader().run(doc, config=config) else: doc = copy.deepcopy(doc) - if config.LOG and not log: + if config.LOG and not log: # python:S3776 log = init_sed_document_log(doc) verbose = config.VERBOSE @@ -126,7 +126,7 @@ def exec_task(task, variables, preprocessed_task=None, log=None, config=None, ** exceptions = [] # execute tasks - if not doc.tasks: + if not doc.tasks: # python:S3776 warn('SED document does not describe any tasks.', NoTasksWarning) # TODO: initialize reports with their eventual shapes; this requires individual simulation tools to pass