Implement improved configuration for SPARTA reductions

Close #73

examples/sparta_reduction.py

@@ -7,6 +7,20 @@
 #%% init reduction
 reduction = SPARTA.Reduction('/Users/avigan/data/sphere-test-target/SPARTA/', log_level='info')
 
+###############################################################################
+# It is possible to provide a default JSON configuration file to set some (or #
+# all) of the reduction parameters to a default value different from the ones #
+# hard-coded in the sphere package. This is done with the keyword:            #
+#   user_config='... path to the file ...'                                    #
+# The increasing priority for setting reduction parameters is the following:  #
+#   0- default values hard-coded in the sphere package                        #
+#   1- values contained in the file pointed by the user_config keyword, if a  #
+#      file path is provided and exists                                       #
+#   2- values contained in a reduction_config.json file left in the reduction #
+#      directory by a previous reduction                                      #
+#   3- values manually set by the user (see examples below)                   #
+###############################################################################
+
 #%% configuration
 reduction.config['misc_plot'] = True
 reduction.show_config()

sphere/SPARTA.py

@@ -57,7 +57,7 @@ class Reduction(object):
     # Constructor
     ##################################################
 
-    def __new__(cls, path, log_level='info', sphere_handler=None):
+    def __new__(cls, path, log_level='info', user_config=None, sphere_handler=None):
         '''
         Custom instantiation for the class
 
@@ -71,9 +71,13 @@ def __new__(cls, path, log_level='info', sphere_handler=None):
         path : str
             Path to the directory containing the dataset
 
-        level : {'debug', 'info', 'warning', 'error', 'critical'}
+        log_level : {'debug', 'info', 'warning', 'error', 'critical'}
             The log level of the handler
 
+        user_config : str
+            Path to a user-provided configuration. Default is None, i.e. the
+            reduction will use the package default configuration parameters
+
         sphere_handler : log handler
             Higher-level SPHERE.Dataset log handler
         '''
@@ -130,19 +134,27 @@ def __new__(cls, path, log_level='info', sphere_handler=None):
         #
         reduction._logger.debug('> read default configuration')
         configfile = f'{Path(sphere.__file__).parent}/instruments/{reduction._instrument}.ini'
-        config = configparser.ConfigParser()
+        cfgparser = configparser.ConfigParser()
 
         reduction._logger.debug('Read configuration')
-        config.read(configfile)
+        cfgparser.read(configfile)
 
         # reduction parameters
-        reduction._config = dict(config.items('reduction'))
-        for key, value in reduction._config.items():
+        cfg = {}
+        items = dict(cfgparser.items('reduction'))
+        for key, value in items.items():
             try:
                 val = eval(value)
             except NameError:
                 val = value
-            reduction._config[key] = val
+            cfg[key] = val
+        reduction._config = utils.Configuration(reduction._path, reduction._logger, cfg)
+
+        # load user-provided default configuration parameters
+        if user_config:
+            user_config = Path(user_config)
+
+            reduction._config.load_from_file(user_config)
 
         #
         # reduction and recipe status
@@ -250,6 +262,9 @@ def _read_info(self):
         # path
         path = self.path
 
+        # load existing configuration
+        self.config.load()
+
         # files info
         fname = path.preproc / 'files.csv'
         if fname.exists():
@@ -372,28 +387,6 @@ def _update_recipe_status(self, recipe, status):
     # Generic class methods
     ##################################################
 
-    def show_config(self):
-        '''
-        Shows the reduction configuration
-        '''
-
-        # dictionary
-        dico = self._config
-
-        # parameters
-        print()
-        print(f'{"Parameter":<30s}Value')
-        print('-'*35)
-        catgs = ['misc', 'clean']
-        for catg in catgs:
-            keys  = [key for key in dico if key.startswith(catg)]
-            for key in keys:
-                print(f'{key:<30s}{dico[key]}')
-            print('-'*35)
-
-        print()
-
-
     def init_reduction(self):
         '''
         Sort files and frames, perform sanity check
@@ -450,7 +443,8 @@ def clean(self):
 
         if config['clean']:
             self.sph_sparta_clean(delete_raw=config['clean_delete_raw'],
-                                  delete_products=config['clean_delete_products'])
+                                  delete_products=config['clean_delete_products'],
+                                  delete_config=config['clean_delete_config'])
 
     def full_reduction(self):
         '''
@@ -1406,7 +1400,7 @@ def sph_sparta_plot(self):
         self._status = sphere.COMPLETE
 
 
-    def sph_sparta_clean(self, delete_raw=False, delete_products=False):
+    def sph_sparta_clean(self, delete_raw=False, delete_products=False, delete_config=False):
         '''
         Clean everything except for raw data and science products (by default)
 
@@ -1417,6 +1411,9 @@ def sph_sparta_clean(self, delete_raw=False, delete_products=False):
 
         delete_products : bool
             Delete science products. Default is False
+
+        delete_config : bool
+            Delete configuration file. Default is False
         '''
 
         self._logger.info('Clean reduction data')
@@ -1426,42 +1423,12 @@ def sph_sparta_clean(self, delete_raw=False, delete_products=False):
                                          self.recipe_requirements, logger=self._logger):
             return
 
-        # parameters
-        path = self.path
+        # remove sub-directories
+        self.path.remove(delete_raw=delete_raw, delete_products=delete_products, logger=self._logger)
 
-        # tmp
-        if path.tmp.exists():
-            self._logger.debug(f'> remove {path.tmp}')
-            shutil.rmtree(path.tmp, ignore_errors=True)
-
-        # sof
-        if path.sof.exists():
-            self._logger.debug(f'> remove {path.sof}')
-            shutil.rmtree(path.sof, ignore_errors=True)
-
-        # calib
-        if path.calib.exists():
-            self._logger.debug(f'> remove {path.calib}')
-            shutil.rmtree(path.calib, ignore_errors=True)
-
-        # preproc
-        if path.preproc.exists():
-            self._logger.debug(f'> remove {path.preproc}')
-            shutil.rmtree(path.preproc, ignore_errors=True)
-
-        # raw
-        if delete_raw:
-            if path.raw.exists():
-                self._logger.debug(f'> remove {path.raw}')
-                self._logger.warning('   ==> delete raw files')
-                shutil.rmtree(path.raw, ignore_errors=True)
-
-        # products
-        if delete_products:
-            if path.products.exists():
-                self._logger.debug(f'> remove {path.products}')
-                self._logger.warning('   ==> delete products')
-                shutil.rmtree(path.products, ignore_errors=True)
+        # remove config
+        if delete_config:
+            self.config._file.unlink()
 
         # update recipe execution
         self._update_recipe_status('sph_sparta_clean', sphere.SUCCESS)

sphere/instruments/SPARTA.ini

@@ -21,4 +21,4 @@ misc_query_timeout   = 5
 clean = True
 clean_delete_raw = False
 clean_delete_products = False
-
+clean_delete_config = False