From 6be2b1a7ecab6a51402d466018359343f790fbb3 Mon Sep 17 00:00:00 2001 From: Arthur Vigan Date: Mon, 13 Jun 2022 14:06:37 +0200 Subject: [PATCH] Implement improved configuration for SPARTA reductions Close #73 --- examples/sparta_reduction.py | 14 +++++ sphere/SPARTA.py | 99 ++++++++++++----------------------- sphere/instruments/SPARTA.ini | 2 +- 3 files changed, 48 insertions(+), 67 deletions(-) diff --git a/examples/sparta_reduction.py b/examples/sparta_reduction.py index 88caede..e753291 100644 --- a/examples/sparta_reduction.py +++ b/examples/sparta_reduction.py @@ -7,6 +7,20 @@ #%% init reduction reduction = SPARTA.Reduction('/Users/avigan/data/sphere-test-target/SPARTA/', log_level='info') +############################################################################### +# It is possible to provide a default JSON configuration file to set some (or # +# all) of the reduction parameters to a default value different from the ones # +# hard-coded in the sphere package. This is done with the keyword: # +# user_config='... path to the file ...' # +# The increasing priority for setting reduction parameters is the following: # +# 0- default values hard-coded in the sphere package # +# 1- values contained in the file pointed by the user_config keyword, if a # +# file path is provided and exists # +# 2- values contained in a reduction_config.json file left in the reduction # +# directory by a previous reduction # +# 3- values manually set by the user (see examples below) # +############################################################################### + #%% configuration reduction.config['misc_plot'] = True reduction.show_config() diff --git a/sphere/SPARTA.py b/sphere/SPARTA.py index 32fc8b8..85c6d5e 100644 --- a/sphere/SPARTA.py +++ b/sphere/SPARTA.py @@ -57,7 +57,7 @@ class Reduction(object): # Constructor ################################################## - def __new__(cls, path, log_level='info', sphere_handler=None): + def __new__(cls, path, log_level='info', user_config=None, sphere_handler=None): ''' Custom instantiation for the class @@ -71,9 +71,13 @@ def __new__(cls, path, log_level='info', sphere_handler=None): path : str Path to the directory containing the dataset - level : {'debug', 'info', 'warning', 'error', 'critical'} + log_level : {'debug', 'info', 'warning', 'error', 'critical'} The log level of the handler + user_config : str + Path to a user-provided configuration. Default is None, i.e. the + reduction will use the package default configuration parameters + sphere_handler : log handler Higher-level SPHERE.Dataset log handler ''' @@ -130,19 +134,27 @@ def __new__(cls, path, log_level='info', sphere_handler=None): # reduction._logger.debug('> read default configuration') configfile = f'{Path(sphere.__file__).parent}/instruments/{reduction._instrument}.ini' - config = configparser.ConfigParser() + cfgparser = configparser.ConfigParser() reduction._logger.debug('Read configuration') - config.read(configfile) + cfgparser.read(configfile) # reduction parameters - reduction._config = dict(config.items('reduction')) - for key, value in reduction._config.items(): + cfg = {} + items = dict(cfgparser.items('reduction')) + for key, value in items.items(): try: val = eval(value) except NameError: val = value - reduction._config[key] = val + cfg[key] = val + reduction._config = utils.Configuration(reduction._path, reduction._logger, cfg) + + # load user-provided default configuration parameters + if user_config: + user_config = Path(user_config) + + reduction._config.load_from_file(user_config) # # reduction and recipe status @@ -250,6 +262,9 @@ def _read_info(self): # path path = self.path + # load existing configuration + self.config.load() + # files info fname = path.preproc / 'files.csv' if fname.exists(): @@ -372,28 +387,6 @@ def _update_recipe_status(self, recipe, status): # Generic class methods ################################################## - def show_config(self): - ''' - Shows the reduction configuration - ''' - - # dictionary - dico = self._config - - # parameters - print() - print(f'{"Parameter":<30s}Value') - print('-'*35) - catgs = ['misc', 'clean'] - for catg in catgs: - keys = [key for key in dico if key.startswith(catg)] - for key in keys: - print(f'{key:<30s}{dico[key]}') - print('-'*35) - - print() - - def init_reduction(self): ''' Sort files and frames, perform sanity check @@ -450,7 +443,8 @@ def clean(self): if config['clean']: self.sph_sparta_clean(delete_raw=config['clean_delete_raw'], - delete_products=config['clean_delete_products']) + delete_products=config['clean_delete_products'], + delete_config=config['clean_delete_config']) def full_reduction(self): ''' @@ -1406,7 +1400,7 @@ def sph_sparta_plot(self): self._status = sphere.COMPLETE - def sph_sparta_clean(self, delete_raw=False, delete_products=False): + def sph_sparta_clean(self, delete_raw=False, delete_products=False, delete_config=False): ''' Clean everything except for raw data and science products (by default) @@ -1417,6 +1411,9 @@ def sph_sparta_clean(self, delete_raw=False, delete_products=False): delete_products : bool Delete science products. Default is False + + delete_config : bool + Delete configuration file. Default is False ''' self._logger.info('Clean reduction data') @@ -1426,42 +1423,12 @@ def sph_sparta_clean(self, delete_raw=False, delete_products=False): self.recipe_requirements, logger=self._logger): return - # parameters - path = self.path + # remove sub-directories + self.path.remove(delete_raw=delete_raw, delete_products=delete_products, logger=self._logger) - # tmp - if path.tmp.exists(): - self._logger.debug(f'> remove {path.tmp}') - shutil.rmtree(path.tmp, ignore_errors=True) - - # sof - if path.sof.exists(): - self._logger.debug(f'> remove {path.sof}') - shutil.rmtree(path.sof, ignore_errors=True) - - # calib - if path.calib.exists(): - self._logger.debug(f'> remove {path.calib}') - shutil.rmtree(path.calib, ignore_errors=True) - - # preproc - if path.preproc.exists(): - self._logger.debug(f'> remove {path.preproc}') - shutil.rmtree(path.preproc, ignore_errors=True) - - # raw - if delete_raw: - if path.raw.exists(): - self._logger.debug(f'> remove {path.raw}') - self._logger.warning(' ==> delete raw files') - shutil.rmtree(path.raw, ignore_errors=True) - - # products - if delete_products: - if path.products.exists(): - self._logger.debug(f'> remove {path.products}') - self._logger.warning(' ==> delete products') - shutil.rmtree(path.products, ignore_errors=True) + # remove config + if delete_config: + self.config._file.unlink() # update recipe execution self._update_recipe_status('sph_sparta_clean', sphere.SUCCESS) diff --git a/sphere/instruments/SPARTA.ini b/sphere/instruments/SPARTA.ini index b2e9f74..820470e 100644 --- a/sphere/instruments/SPARTA.ini +++ b/sphere/instruments/SPARTA.ini @@ -21,4 +21,4 @@ misc_query_timeout = 5 clean = True clean_delete_raw = False clean_delete_products = False - +clean_delete_config = False