Started work on inform stage and yaml config stuff

src/lephare/__init__.py

@@ -26,5 +26,6 @@
 from .filterSvc import *
 from .mag_gal import *
 from .magSvc import *
+from .prepare import *
 from .sedtolib import *
 from .zphota import *

src/lephare/filterSvc.py

@@ -220,3 +220,37 @@
         # save the SVO additional info in an instance property
         flt_obj.svo_params = param_dict
         return flt_obj
+
+    @classmethod
+    def from_keymap(cls, keymap):
+        """Load filter from a config keymap.
+
+        Parameters
+        ----------
+        keymap : dict of lephare.keyword
+            The config keymap
+
+        Returns
+        -------
+        flt_array : list of `lephare.flt`
+            List of filters.
+        """
+        keymap["FILTER_REP"].value = keymap["FILTER_REP"].value.replace("$LEPHAREDIR", LEPHAREDIR)
+
+        filter_list = keymap["FILTER_LIST"].value.split(",")
+        filter_calib = keymap["FILTER_CALIB"].value.split(",")
+        filter_trans = keymap["TRANS_TYPE"].value.split(",")
+        if len(filter_trans) == 1:
+            filter_trans = len(filter_list) * filter_trans
+        elif len(filter_trans) != len(filter_list):
+            raise RuntimeError("FILTER_LIST and FILTER_TRANS do not have the same size")
+        if len(filter_calib) == 1:
+            filter_calib = len(filter_list) * filter_calib
+        elif len(filter_calib) != len(filter_list):
+            raise RuntimeError("FILTER_LIST and FILTER_CALIB do not have the same size")
+        flt_array = []
+        for i in range(len(filter_list)):
+            name = os.path.join(keymap["FILTER_REP"].value, filter_list[i])
+            oneflt = flt(i, name, int(filter_trans[i]), int(filter_calib[i]))
+            flt_array.append(oneflt)
+        return flt_array

src/lephare/prepare.py

@@ -0,0 +1,125 @@
+import datetime
+import os
+
+import yaml
+
+import lephare as lp
+
+__all__ = ["prepare", "overwrite_config", "read_yaml_config", "write_yaml_config", "write_para_config"]
+
+
+def prepare(config, star_config=None, gal_config=None, qso_config=None):
+    """Run the prepare stages of LePHARE
+
+    In order to run "zphota" we must create the filter files, run sedtolib to
+    create the SED libraries, and finally run mag_gal, to create the
+    magnitude libararies. We abstract these tasks into a single prepare stage.
+
+    We overide the config for each type if distinct config set. If no overide
+    configs are set we use the same for each type.
+
+    Parameters
+    ==========
+    config : dict of lephare.keyword
+        The config base to run all tasks
+    star_config : dict of lephare.keyword or None
+        Config values to override for stars
+    gal_config : dict of lephare.keyword or None
+        Config values to override for galaxies
+    qso_config : dict of lephare.keyword or None
+        Config values to override for QSO.
+    """
+    object_types = {"STAR": star_config, "GAL": gal_config, "QSO": qso_config}
+    # Run the filter command
+    # load filters from config
+    filter_lib = lp.FilterSvc.from_keymap(config)
+    # Get location to store filter files
+    filter_output = os.path.join(os.environ["LEPHAREWORK"], "filt", config["FILTER_FILE"].value)
+    # Write filter files
+    lp.write_output_filter(filter_output + ".dat", filter_output + ".doc", filter_lib)
+    # # Write config to work directory
+    write_yaml_config(config, f"{filter_output}_config.yaml")
+
+    for object_type in object_types:
+        updated_config = overwrite_config(config, object_types[object_type])
+        # Write the updated config
+        sed_out_name = f"{updated_config[f'{object_type}_LIB'].value}_{object_type.lower()}_config.yaml"
+        sed_output = os.path.join(os.environ["LEPHAREWORK"], "lib_bin", sed_out_name)
+        mag_out_name = f"{updated_config[f'{object_type}_LIB_OUT'].value}_{object_type.lower()}_config.yaml"
+        mag_output = os.path.join(os.environ["LEPHAREWORK"], "lib_mag", mag_out_name)
+        # Run sedtolib
+        sedlib = lp.Sedtolib(config_keymap=updated_config)
+        list_loc = os.path.join(lp.LEPHAREDIR, updated_config[f"{object_type}_SED"].value)
+        sedtolib_kwargs = {f"{object_type.lower()}_sed": list_loc}
+        print(sedtolib_kwargs)
+        sedlib.run(typ=object_type, **sedtolib_kwargs)
+        write_yaml_config(updated_config, sed_output)
+        # Run mag_gal
+        maglib = lp.MagGal(config_keymap=updated_config)
+        maglib.run(typ=object_type)
+        write_yaml_config(updated_config, mag_output)
+
+
+def overwrite_config(config1, config2):
+    """Check that two config can be safely broadcast for a joint run"""
+    if config2 is None:
+        return config1
+    config = config1
+    # Redshift grid must be idenitical
+    # assert True
+    # filters must be indentical
+    # assert True
+    for k in config2:
+        config[k] = config2[k]
+    return config
+
+
+def read_yaml_config(yaml_file_path):
+    """Open a standard yaml file and render it as a dictionary of keywords
+
+    Parameters
+    ==========
+    yaml_file_path : str
+        Path to input yaml file.
+    """
+    with open(yaml_file_path, "r") as file:
+        config = yaml.safe_load(file)
+    for key in config:
+        config[key] = lp.keyword(key, config[key])
+    return config
+
+
+def write_yaml_config(keymap, yaml_file_path):
+    """Write a dictionary of keywords to a yaml file
+
+    Parameters
+    ==========
+    keymap : dict of lephare.keyword
+        The dictionary of keywords to be written to yaml.
+    yaml_file_path : str
+        Path to output yaml file.
+    """
+    config_dict = {}
+    for k in keymap:
+        config_dict[keymap[k].name] = keymap[k].value
+    with open(yaml_file_path, "w") as yaml_file:
+        yaml.dump(config_dict, yaml_file)
+
+
+def write_para_config(keymap, para_file_path):
+    """Write a dictionary of keywords to a para file
+
+    Parameters
+    ==========
+    keymap : dict of lephare.keyword
+        The dictionary of keywords to be written to yaml.
+    para_file_path : str
+        Path to output para file.
+    """
+    now = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
+    para_contents = f"# File written automatically at {now}\n"
+    for k in keymap:
+        para_contents += f"{k} {keymap[k].value}\n"
+    with open(para_file_path, "w") as file_handle:
+        file_handle.write(para_contents)
+        file_handle.close()

tests/data/alloutputkeys.txt

@@ -0,0 +1,134 @@
+IDENT                   str           spec
+Z_BEST                  float	      zgmin[0]
+Z_BEST68_LOW            float	      zgmin[1]
+Z_BEST68_HIGH           float	      zgmin[2]
+Z_BEST90_LOW            float	      zgmin[3]
+Z_BEST90_HIGH           float	      zgmin[4]
+Z_BEST99_LOW            float	      zgmin[5]
+Z_BEST99_HIGH           float	      zgmin[6]
+Z_MED                   float	      zgmed[0]
+Z_MED68_LOW		float	      zgmed[1]
+Z_MED68_HIGH		float	      zgmed[2]
+Z_MED90_LOW		float	      zgmed[3]
+Z_MED90_HIGH		float	      zgmed[4]
+Z_MED99_LOW		float	      zgmed[5]
+Z_MED99_HIGH		float	      zgmed[6]
+Z_MODE                  float	      zgmode[0]
+Z_MODE68_LOW		float	      zgmode[1]
+Z_MODE68_HIGH		float	      zgmode[2]
+Z_MODE90_LOW		float	      zgmode[3]
+Z_MODE90_HIGH		float	      zgmode[4]
+Z_MODE99_LOW		float	      zgmode[5]
+Z_MODE99_HIGH		float	      zgmode[6]
+MASS_BEST               float         results["MASS_BEST"]
+SFR_BEST                float         results["SFR_BEST"]
+SSFR_BEST               float         results["SFR_BEST"]
+LDUST_BEST              float         results["LDUST_BEST"]
+AGE_BEST                float         results["AGE_BEST"]
+EBV_BEST                float         results["EBV_BEST"]
+EXTLAW_BEST             float         results["EXTLAW_BEST"]
+LUM_NUV_BEST            float         results["LUM_NUV_BEST"]
+LUM_R_BEST              float         results["LUM_R_BEST"]
+LUM_K_BEST              float         results["LUM_K_BEST"]
+LUM_TIR_BEST            float         results["LUM_TIR_BEST"]
+MOD_BEST                int           imasmin[0]
+#PDZ_BEST
+CHI_BEST                float         chimin[0]
+SCALE_BEST              float         dmmin[0]
+CONTEXT                 long          cont
+NBAND_USED              int           nbused
+NBAND_ULIM              int           nbul
+ZSPEC                   float         zs
+MAG_OBS()               float         mab
+ERR_MAG_OBS()           float         msab
+MAG_MOD()               float         magm
+MAG_PRED()              float         magPred
+ABSMAG_PRED()           float         absmagPred
+ZQ_BEST                 float	      zqmin[0]
+ZQ_BEST68_LOW           float	      zqmin[1]
+ZQ_BEST68_HIGH          float	      zqmin[2]
+ZQ_BEST90_LOW           float	      zqmin[3]
+ZQ_BEST90_HIGH          float	      zqmin[4]
+ZQ_BEST99_LOW           float	      zqmin[5]
+ZQ_BEST99_HIGH          float	      zqmin[6]
+ZQ_MED                  float	      zqmed[0]
+ZQ_MED68_LOW		float	      zqmed[1]
+ZQ_MED68_HIGH		float	      zqmed[2]
+ZQ_MED90_LOW		float	      zqmed[3]
+ZQ_MED90_HIGH		float	      zqmed[4]
+ZQ_MED99_LOW		float	      zqmed[5]
+ZQ_MED99_HIGH		float	      zqmed[6]
+ZQ_MODE                 float	      zqmode[0]
+ZQ_MODE68_LOW		float	      zqmode[1]
+ZQ_MODE68_HIGH		float	      zqmode[2]
+ZQ_MODE90_LOW		float	      zqmode[3]
+ZQ_MODE90_HIGH		float	      zqmode[4]
+ZQ_MODE99_LOW		float	      zqmode[5]
+ZQ_MODE99_HIGH		float	      zqmode[6]
+CHI_QSO                 float         chimin[1] 
+MOD_QSO                 int           imasmin[1]
+CHI_STAR                float         chimin[2]
+MOD_STAR                int           imasmin[2]
+MOD_SEC                 int           zsecMod
+EXTLAW_SEC              int           zsecExtlaw
+Z_SEC                   float         zsec
+EBV_SEC                 float         zsecEbv
+PDZ_SEC                 float         zsecProb
+CHI_SEC                 float         zsecChi2
+SCALE_SEC               float         zsecScale
+AGE_SEC                 float         zsecAge
+AGE_MED                 float         agemed[0]
+AGE_INF                 float         agemed[1]
+AGE_SUP                 float         agemed[2]
+LDUST_MED               float         Ldustmed[0]
+LDUST_INF		float         Ldustmed[1]
+LDUST_SUP		float         Ldustmed[2]
+LUM_TIR_MED             float         LIRmed[0]
+LUM_TIR_INF		float         LIRmed[1]
+LUM_TIR_SUP		float         LIRmed[2]
+MASS_MED                float         massmed[0]
+MASS_INF		float         massmed[1]
+MASS_SUP		float         massmed[2]
+#EBV_MED                 float         med[0]
+#EBV_INF                 float         med[1]
+#EBV_SUP                 float         med[2]
+SFR_MED                 float         SFRmed[0]
+SFR_INF			float         SFRmed[1]
+SFR_SUP			float         SFRmed[2]
+SSFR_MED                float         sSFRmed[0]
+SSFR_INF		float         sSFRmed[1]
+SSFR_SUP		float         sSFRmed[2]
+COL1_MED                float         col1med[0]
+COL1_INF		float         col1med[1]
+COL1_SUP		float         col1med[2]
+COL2_MED                float         col2med[0]
+COL2_INF		float         col2med[1]
+COL2_SUP		float         col2med[2]
+MREF_MED                float         Mrefmed[0]
+MREF_INF		float         Mrefmed[1]
+MREF_SUP		float         Mrefmed[2]
+LIMITS_ZMAX             float         limits_zmax
+LIMITS_MFAINT           float         limits_Mfaint
+EM_FLUX_LYA             float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_OII             float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_HB              float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_OIIIA           float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_OIIIB           float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_HA              float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_SIIIA           float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX_SIIIB           float         results_emission_lines["EM_FLUX_LYA"]
+EM_FLUX()               float          fluxEL_SED
+EM_EW_LYA               float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_OII               float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_HB                float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_OIIIA             float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_OIIIB             float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_HA                float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_SIIIA             float         results_emission_lines["EM_FLUX_LYA"]
+EM_EW_SIIIB             float         results_emission_lines["EM_FLUX_LYA"]
+#MABS_FLUX
+MABS_FILT()             float         absfilt
+K_COR()                 float         kap
+MAG_ABS()               float         mabs
+EMAG_ABS()              float         emabs
+STRING_INPUT            str           str_inp

tests/data/examples/COSMOS.para

@@ -4,22 +4,26 @@
 ##############################################################################################
 #
 #-------------------    STELLAR LIBRARY (ASCII SEDs)        ---------------------------       
-STAR_SED        STAR_SWIRE.list         # STAR list (full path)
+STAR_SED        sed/STAR/ONE_SED.list         # STAR list (full path)
+#STAR_SED        sed/STAR/STAR_MOD_ALL.list         # STAR list (full path)
 STAR_LIB	LIB_STAR		# Binary STAR LIBRARY (-> $ZPHOTWORK/lib_bin/*)
 STAR_FSCALE	3.432E-09		# Arbitrary Flux Scale 
 #
 #-------------------    QSO LIBRARY (ASCII SEDs)            ---------------------------  
-QSO_SED		AGN_LONSDALE.list       # QSO list (full path)
+QSO_SED		sed/QSO/ONE_SED.list       # QSO list (full path)
+#QSO_SED		sed/QSO/SALVATO09/AGN_MOD.list       # QSO list (full path)
 QSO_LIB		LIB_QSO			# Binary QSO LIBRARY (-> $ZPHOTWORK/lib_bin/*)
 QSO_FSCALE	1.			# Arbitrary Flux Scale 
 #
 #-------------------  GALAXY LIBRARY (ASCII or BINARY SEDs) ---------------------------
-GAL_SED		 CE_MOD.list            # GALAXMuzzin09_SEDY list (full path)
+GAL_SED		 sed/GAL/ONE_SED.list            # GALAXMuzzin09_SEDY list (full path)
+#GAL_SED		 examples/COSMOS_MOD.list            # GALAXMuzzin09_SEDY list (full path)
 GAL_LIB	         LIB_CE	                # Binary GAL LIBRARY (-> $ZPHOTWORK/lib_bin/*)
 GAL_FSCALE      1.                      # Arbitrary Flux Scale
 #SEL_AGE 	/data/zphot_vers25_03_03/sed/GAL/HYPERZ/AGE_GISSEL_HZ.dat	# List of Age for GISSEL(full path)
 AGE_RANGE  0.,15.e9                                     # Age Min-Max in yr
 
+
 
 #
 ##############################################################################################
@@ -29,7 +33,7 @@ AGE_RANGE  0.,15.e9                                     # Age Min-Max in yr
 
 # 
 FILTER_REP   $LEPHAREDIR/filt           # Repository in which the filters are stored
-FILTER_LIST   subaru/IB527.pb
+FILTER_LIST   subaru/IB527.pb 
 # FILTER_LIST   cosmos/u_cfht.lowres,cosmos/u_new.pb,hsc/gHSC.pb,hsc/rHSC.pb,hsc/iHSC.pb,hsc/zHSC.pb,hsc/yHSC.pb,vista/Y.lowres,vista/J.lowres,vista/H.lowres,vista/K.lowres,cosmos/IB427.lowres,cosmos/IB464.lowres,cosmos/IB484.lowres,cosmos/IB505.lowres,cosmos/IB527.lowres,cosmos/IB574.lowres,cosmos/IB624.lowres,cosmos/IB679.lowres,cosmos/IB709.lowres,cosmos/IB738.lowres,cosmos/IB767.lowres,cosmos/IB827.lowres,cosmos/NB711.lowres,cosmos/NB816.lowres,vista/NB118.lowres,cosmos/irac_ch1.lowres,cosmos/irac_ch2.lowres,cosmos/irac_ch3.lowres,cosmos/irac_ch4.lowres 
 TRANS_TYPE	 1			# TRANSMISSION TYPE
                                         # 0[-def]: Energy, 1: Nb of photons
@@ -63,7 +67,7 @@ GAL_LIB_OUT	CE_COSMOS	# Output GALAXY LIBRARY  (-> $ZPHOTWORK/lib_mag/*)
 #------------------   MAG + Z_STEP + COSMO + EXTINCTION   -----------------------------
 MAGTYPE         AB		# Magnitude type (AB or VEGA)
 ZGRID_TYPE      0               # Define the kind of redshift grid (0: linear ; 1: dz*(1+z)) 
-Z_STEP 		0.01,0.,7. 	# dz, zmin, zmax 
+Z_STEP 		0.1,0.,5. 	# dz, zmin, zmax 
 COSMOLOGY	70,0.3,0.7	# H0,om0,lbd0    (if lb0>0->om0+lbd0=1)
 MOD_EXTINC 	0,0		# model range for extinction 
 EXTINC_LAW	SB_calzetti.dat	# ext. law (in  $ZPHOTDIR/ext/*)
@@ -92,7 +96,7 @@ CAT_LINES       0,1000000000     #  MIN and MAX RANGE of ROWS used in input cat
 CAT_TYPE	LONG	  	 # Input Format    (LONG,SHORT-def)
 GLB_CONTEXT	0		 # Overwrite Context (Sum 2^n; n=0->nbd-1, 0->all bands, -1[-def] used context per object) 
 FORB_CONTEXT	-1		 # Not consider these filters in the fit  (Sum 2^n; n=0->nbd-1) 
-ERR_SCALE       0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.05,0.05,0.05,0.05,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.05,0.05,0.05,0.05,0.1,0.2,0.3  # Systematic errors per band
+ERR_SCALE       0.02 # Systematic errors per band
 ERR_FACTOR      1.5              # Multiply all the flux uncertainties by this factor
 ADD_EMLINES     0,10000          # Range of template in which we add emission lines
 
@@ -119,7 +123,7 @@ RM_DISCREPANT_BD 500             # Threshold in chi2 to consider. Remove <3 band
 #------------------      Priors                       --------------------------------- 
 MAG_ABS 	-24,-5  	# Mabs_min , Mabs_max [0,0-def]
 MAG_ABS_QSO 	-30,-10  	# Mabs_min , Mabs_max for QSO library [0,0-def]
-MAG_REF 	3		# Reference number for band used by Mag_abs
+MAG_REF 	1		# Reference number for band used by Mag_abs
 Z_RANGE        0.,99.99          # Z min-max used for the Galaxy library 
 EBV_RANGE      0,9               # E(B-V) MIN-MAX RANGE of E(B-V) used  
 #NZ_PRIOR       4,5                # I Band for prior on N(z), the second number is the band to be used if the first is missing.
@@ -162,7 +166,7 @@ CHI2_OUT        NO               # output file with all values : z,mod,chi2,E(B-
 #-------------------   ADAPTIVE METHOD using Z spectro sample -----------------
 #
 AUTO_ADAPT	NO		 # Adapting method with spectro [NO-def]
-ADAPT_BAND 	5		 # Reference band, band1, band2 for color 
+ADAPT_BAND 	1		 # Reference band, band1, band2 for color 
 ADAPT_LIM       1.5,23.0         # Mag limits for spectro in Ref band [18,21.5-def]
 ADAPT_CONTEXT  -1                # Context for bands used for training
                                  # If -1[-def] the ones given by the normal context