Add code to retrieve throughputs versions, write to flux file metadata

Add utility script to dump metadata from a parquet file

skycatalogs/catalog_creator.py

@@ -727,21 +727,28 @@ def create_galaxy_flux_catalog(self, config_file=None):
         from .skyCatalogs import open_catalog
         self._sed_gen = None
 
-        file_metadata = assemble_file_metadata(self._pkg_root,
-                                               run_options=self._run_options,
-                                               flux_file=True)
-        self._gal_flux_schema =\
-            make_galaxy_flux_schema(self._logname, self._galaxy_type,
-                                    include_roman_flux=self._include_roman_flux,
-                                    metadata_input=file_metadata)
-        self._gal_flux_needed = [field.name for field in self._gal_flux_schema]
-
         if not config_file:
             config_file = self.get_config_path()
         if not self._cat:
             self._cat = open_catalog(config_file,
                                      skycatalog_root=self._skycatalog_root)
 
+        # Throughput versions for fluxes included
+        thru_v = {'lsst_throughputs_version': self._cat._lsst_thru_v}
+        if self._include_roman_flux:
+            thru_v['roman_throughputs_version'] = self._cat._roman_thru_v
+
+        file_metadata = assemble_file_metadata(
+            self._pkg_root,
+            run_options=self._run_options,
+            flux_file=True,
+            throughputs_versions=thru_v)
+        self._gal_flux_schema =\
+            make_galaxy_flux_schema(self._logname, self._galaxy_type,
+                                    include_roman_flux=self._include_roman_flux,
+                                    metadata_input=file_metadata)
+        self._gal_flux_needed = [field.name for field in self._gal_flux_schema]
+
         # Might also open a main file and read its metadata to be sure
         # the value for stride is correct.
         # Especially need to do this if we want to support making flux
@@ -1001,7 +1008,7 @@ def create_pointsource_flux_catalog(self, config_file=None):
         Parameters
         ----------
         Path to config created in first stage so we can find the main
-        galaxy files.
+        star files
 
         Return
         ------
@@ -1010,19 +1017,22 @@ def create_pointsource_flux_catalog(self, config_file=None):
 
         from .skyCatalogs import open_catalog
 
-        file_metadata = assemble_file_metadata(self._pkg_root,
-                                               run_options=self._run_options,
-                                               flux_file=True)
-
-        self._ps_flux_schema = make_star_flux_schema(self._logname,
-                                                     metadata_input=file_metadata)
         if not config_file:
             config_file = self.get_config_path()
 
         # Always open catalog. If it was opened for galaxies earlier
         # it won't know about star files.
         self._cat = open_catalog(config_file,
                                  skycatalog_root=self._skycatalog_root)
+        thru_v = {'lsst_throughputs_version': self._cat._lsst_thru_v}
+        file_metadata = assemble_file_metadata(
+            self._pkg_root,
+            run_options=self._run_options,
+            flux_file=True,
+            throughputs_versions=thru_v)
+
+        self._ps_flux_schema = make_star_flux_schema(self._logname,
+                                                     metadata_input=file_metadata)
 
         self._flux_template = self._cat.raw_config['object_types']['star']['flux_file_template']
 

skycatalogs/objects/base_object.py

@@ -1,7 +1,6 @@
 from collections.abc import Sequence, Iterable
 from collections import namedtuple
 import os
-import itertools
 import numpy as np
 import galsim
 from galsim.roman import longwave_bands as roman_longwave_bands
@@ -30,7 +29,7 @@
 def load_lsst_bandpasses():
     '''
     Read in lsst bandpasses from standard place, trim, and store in global dict
-    Returns: The dict
+    Returns: The dict and throughputs version
     '''
     global lsst_bandpasses
     lsst_bandpasses = dict()
@@ -51,6 +50,11 @@ def load_lsst_bandpasses():
             else:
                 # logger.info('Using galsim built-in bandpasses')
                 bp_dir = None
+    if bp_dir:
+        with open(os.path.join(bp_dir, 'version_info')) as f:
+            version = f.read().strip()
+    else:
+        version = 'galsim_builtin'
 
     for band in LSST_BANDS:
         if bp_dir:
@@ -70,7 +74,7 @@ def load_lsst_bandpasses():
         bp = bp.withZeropoint('AB')
         lsst_bandpasses[band] = bp
 
-    return lsst_bandpasses
+    return lsst_bandpasses, version
 
 
 def load_roman_bandpasses():
@@ -80,7 +84,7 @@ def load_roman_bandpasses():
     '''
     global roman_bandpasses
     roman_bandpasses = roman_getBandpasses()
-    return roman_bandpasses
+    return roman_bandpasses, 'galsim_builtin'
 
 
 class BaseObject(object):
@@ -600,7 +604,7 @@ def __getitem__(self, key):
             return [self.__getitem__(i) for i in range(self.__len__())[key]]
 
         elif type(key) == tuple and isinstance(key[0], Iterable):
-            return[self.__getitem__(i) for i in key[0]]
+            return [self.__getitem__(i) for i in key[0]]
 
     def get_partition_id(self):
         return self._partition_id
@@ -650,7 +654,7 @@ def append_collection(self, coll):
         if isinstance(coll, ObjectCollection):
             self._total_len += len(coll)
             self._located.append(LocatedCollection(coll, old, self._total_len))
-        else:  #  list of collections
+        else:  # list of collections
             for c in coll:
                 self._total_len += len(c)
                 self._located.append(LocatedCollection(c, old, self._total_len))

skycatalogs/scripts/dump_meta.py

@@ -0,0 +1,17 @@
+import yaml
+from skycatalogs.utils.config_utils import get_file_metadata
+import sys
+
+# args are path to file to be read and text (yaml) file to dump it to
+
+if len(sys.argv) == 1:
+    print('to invoke: \n   python dump_meta.py path-to-input-parquet [path-to-output-text]')
+    print('If no output filepath is supplied, output will go to std out')
+    exit(0)
+data = get_file_metadata(sys.argv[1])
+
+if len(sys.argv) > 2:
+    with open(sys.argv[2], mode='w') as f:
+        yaml.dump(data, f)
+else:
+    print(yaml.dump(data))

skycatalogs/skyCatalogs.py

@@ -795,12 +795,14 @@ def open_catalog(config_file, mp=False, skycatalog_root=None, verbose=False):
     -------
     SkyCatalog
     '''
-    # Get bandpasses in case we need to compute fluxes
-    _ = load_lsst_bandpasses()
-    _ = load_roman_bandpasses()
 
     from skycatalogs.utils.config_utils import open_config_file
 
     config_dict = open_config_file(config_file)
-    return SkyCatalog(config_dict, skycatalog_root=skycatalog_root, mp=mp,
-                      verbose=verbose)
+    cat = SkyCatalog(config_dict, skycatalog_root=skycatalog_root, mp=mp,
+                     verbose=verbose)
+
+    # Get bandpasses in case we need to compute fluxes
+    _, cat._lsst_thru_v = load_lsst_bandpasses()
+    _, cat._roman_thru_v = load_roman_bandpasses()
+    return cat

skycatalogs/utils/config_utils.py

@@ -421,7 +421,8 @@ def assemble_provenance(pkg_root, inputs={}, run_options=None,
 
 
 def assemble_file_metadata(pkg_root, inputs=None, run_options=None,
-                           flux_file=False):
+                           flux_file=False, throughputs_versions=None):
+    #                      flux_file=False):
     '''
     Assemble the metadata to be included in a skyCatalogs binary data file
     '''
@@ -433,6 +434,10 @@ def assemble_file_metadata(pkg_root, inputs=None, run_options=None,
         to_return['flux_dependencies'] = dict()
         from galsim import version as galsim_version
         to_return['flux_dependencies']['galsim_version'] = galsim_version
+        if throughputs_versions:
+            for k, v in throughputs_versions.items():
+                to_return['flux_dependencies'][k] = v
+
     return to_return