Skip to content

Commit

Permalink
Merge pull request #83 from LSSTDESC/u/jrbogart/diffsky_integrate2
Browse files Browse the repository at this point in the history 
U/jrbogart/diffsky integrate2
  • Loading branch information
@JoanneBogart
JoanneBogart authored Jan 28, 2024
2 parents d0d0d58 + ba80f35 commit a042dcb
Show file tree
Hide file tree
Showing 4 changed files with 299 additions and 259 deletions.
71 changes: 48 additions & 23 deletions skycatalogs/catalog_creator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
_MW_rv_constant = 3.1
_nside_allowed = 2**np.arange(15)


def _get_tophat_info(columns):
'''
Parameters
Expand Down Expand Up @@ -150,7 +151,8 @@ def _generate_subpixel_masks(ra, dec, subpixels, nside=32):
# Collection of galaxy objects for current row group, current pixel
# Used while doing flux computation

def _do_galaxy_flux_chunk(send_conn, galaxy_collection, instrument_needed, l_bnd, u_bnd):
def _do_galaxy_flux_chunk(send_conn, galaxy_collection, instrument_needed,
l_bnd, u_bnd):
'''
output connection
l_bnd, u_bnd demarcates slice to process
Expand Down Expand Up @@ -191,8 +193,8 @@ def __init__(self, parts, area_partition=None, skycatalog_root=None,
output_type='parquet', mag_cut=None,
sed_subdir='galaxyTopHatSED', knots_mag_cut=27.0,
knots=True, logname='skyCatalogs.creator',
pkg_root=None, skip_done=False, flux_only=False,
main_only=False, flux_parallel=16, galaxy_nside=32,
pkg_root=None, skip_done=False, no_main=False,
no_flux=False, flux_parallel=16, galaxy_nside=32,
galaxy_stride=1000000, provenance=None,
dc2=False, sn_object_type='sncosmo', galaxy_type='cosmodc2',
include_roman_flux=False, star_input_fmt='sqlite'):
Expand Down Expand Up @@ -230,8 +232,8 @@ def __init__(self, parts, area_partition=None, skycatalog_root=None,
skip_done If True, skip over files which already exist. Otherwise
(by default) overwrite with new version.
Output info message in either case if file exists.
flux_only Only create flux files, not main files
main_only Only create main files, not flux files
no_main Do not create main files
no_flux Do not create flux files
flux_parallel Number of processes to divide work of computing fluxes
galaxy_nside Healpix configuration value "nside" for galaxy output
galaxy_stride Max number of rows per galaxy row group
Expand All @@ -248,7 +250,7 @@ def __init__(self, parts, area_partition=None, skycatalog_root=None,
"""

_cosmo_cat = 'cosmodc2_v1.1.4_image_addon_knots'
_diffsky_cat = 'roman_rubin_2023_v1.1.1_elais'
_diffsky_cat = 'roman_rubin_2023_v1.1.2_elais'
_star_db = '/global/cfs/cdirs/lsst/groups/SSim/DC2/dc2_stellar_healpixel.db'
_sn_db = '/global/cfs/cdirs/lsst/groups/SSim/DC2/cosmoDC2_v1.1.4/sne_cosmoDC2_v1.1.4_MS_DDF_healpix.db'

Expand Down Expand Up @@ -316,8 +318,8 @@ def __init__(self, parts, area_partition=None, skycatalog_root=None,
self._logname = logname
self._logger = logging.getLogger(logname)
self._skip_done = skip_done
self._flux_only = flux_only
self._main_only = main_only
self._no_main = no_main
self._no_flux = no_flux
self._flux_parallel = flux_parallel
self._galaxy_nside = galaxy_nside
self._provenance = provenance
Expand Down Expand Up @@ -362,14 +364,14 @@ def create(self, catalog_type):
None
"""
if catalog_type == ('galaxy'):
if not self._flux_only:
if not self._no_main:
self.create_galaxy_catalog()
if not self._main_only:
if not self._no_flux:
self.create_galaxy_flux_catalog()
elif catalog_type == ('pointsource'):
if not self._flux_only:
if not self._no_main:
self.create_pointsource_catalog()
if not self._main_only:
if not self._no_flux:
self.create_pointsource_flux_catalog()
else:
raise NotImplementedError(f'CatalogCreator.create: unsupported catalog type {catalog_type}')
Expand Down Expand Up @@ -501,8 +503,8 @@ def create_galaxy_pixel(self, pixel, gal_cat, arrow_schema):
# to_fetch = all columns of interest in gal_cat
non_sed = ['galaxy_id', 'ra', 'dec', 'redshift', 'redshiftHubble',
'peculiarVelocity', 'shear_1', 'shear_2',
'convergence',
'size_bulge_true', 'size_minor_bulge_true', 'sersic_bulge',
'convergence', 'size_bulge_true',
'size_minor_bulge_true', 'sersic_bulge',
'size_disk_true', 'size_minor_disk_true', 'sersic_disk']
if self._dc2:
non_sed += ['ellipticity_1_disk_true_dc2',
Expand Down Expand Up @@ -535,7 +537,7 @@ def create_galaxy_pixel(self, pixel, gal_cat, arrow_schema):
'convergence', 'diskEllipticity1', 'diskEllipticity2',
'spheroidEllipticity1', 'spheroidEllipticity2',
'spheroidHalfLightRadiusArcsec',
'diskHalfLightRadiusArcsec','um_source_galaxy_obs_sm']
'diskHalfLightRadiusArcsec', 'um_source_galaxy_obs_sm']

# df is not a dataframe! It's just a dict
if not self._mag_cut:
Expand All @@ -562,12 +564,14 @@ def create_galaxy_pixel(self, pixel, gal_cat, arrow_schema):
to_rename['ellipticity_2_bulge_true_dc2'] = 'ellipticity_2_bulge_true'

if self._sed_subdir:
# Generate full paths for disk and bulge SED files, even though
# Generate full paths for disk and bulge SED files even though
# we don't actually write the files here
df['bulge_sed_file_path'] =\
generate_sed_path(df['galaxy_id'], self._sed_subdir, 'bulge')
generate_sed_path(df['galaxy_id'], self._sed_subdir,
'bulge')
df['disk_sed_file_path'] =\
generate_sed_path(df['galaxy_id'], self._sed_subdir, 'disk')
generate_sed_path(df['galaxy_id'], self._sed_subdir,
'disk')

if self._knots:
# adjust disk sed; create knots sed
Expand Down Expand Up @@ -605,7 +609,8 @@ def create_galaxy_pixel(self, pixel, gal_cat, arrow_schema):

if self._galaxy_type == 'cosmodc2':
compressed = self._make_tophat_columns(compressed,
sed_disk_names, 'disk')
sed_disk_names,
'disk')
compressed = self._make_tophat_columns(compressed,
sed_bulge_names,
'bulge')
Expand Down Expand Up @@ -647,10 +652,11 @@ def create_galaxy_flux_catalog(self, config_file=None):
'''

from .skyCatalogs import open_catalog
self._sed_gen = None

self._gal_flux_schema = make_galaxy_flux_schema(self._logname,
self._galaxy_type,
include_roman_flux=self._include_roman_flux)
self._gal_flux_schema =\
make_galaxy_flux_schema(self._logname, self._galaxy_type,
include_roman_flux=self._include_roman_flux)
self._gal_flux_needed = [field.name for field in self._gal_flux_schema]

if not config_file:
Expand All @@ -666,6 +672,16 @@ def create_galaxy_flux_catalog(self, config_file=None):
self.object_type = 'galaxy'
if self._galaxy_type == 'diffsky':
self.object_type = 'diffsky_galaxy'
from .diffsky_sedgen import DiffskySedGenerator
# Default values are ok for all the diffsky-specific
# parameters: include_nonLSST_flux, sed_parallel, auto_loop,
# wave_ang_min, wave_ang_max, rel_err, n_per
self._sed_gen = DiffskySedGenerator(logname=self._logname,
galaxy_truth=self._galaxy_truth,
output_dir=self._output_dir,
skip_done=True,
sky_cat=self._cat)

self._flux_template = self._cat.raw_config['object_types'][self.object_type]['flux_file_template']

self._logger.info('Creating galaxy flux files')
Expand Down Expand Up @@ -710,6 +726,14 @@ def _create_galaxy_flux_pixel(self, pixel):
# If there are multiple row groups, each is stored in a separate
# object collection. Need to loop over them
object_list = self._cat.get_object_type_by_hp(pixel, self.object_type)
if len(object_list) == 0:
self._logger.warning(f'Cannot create flux file for pixel {pixel} because main file does not exist or is empty')
return

if self._galaxy_type == 'diffsky':
# Generate SEDs if necessary
self._sed_gen.generate_pixel(pixel)

writer = None
global _galaxy_collection
global _instrument_needed
Expand Down Expand Up @@ -760,7 +784,8 @@ def _create_galaxy_flux_pixel(self, pixel):
# For debugging call directly
proc = Process(target=_do_galaxy_flux_chunk,
name=f'proc_{i}',
args=(conn_wrt, _galaxy_collection, _instrument_needed, lb, u))
args=(conn_wrt, _galaxy_collection,
_instrument_needed, lb, u))
proc.start()
p_list.append(proc)
lb = u
Expand Down
Loading

0 comments on commit a042dcb

Please sign in to comment.