diff --git a/harnessed_jobs/nonlinearity_BOT/v0/modulefile b/harnessed_jobs/nonlinearity_BOT/v0/modulefile new file mode 100644 index 0000000..0be0eec --- /dev/null +++ b/harnessed_jobs/nonlinearity_BOT/v0/modulefile @@ -0,0 +1,3 @@ +#%Module1.0 #-*-tcl-*-# +source "$::env(LCATR_MODULES)/lcatr.tcl" +lcatr_package producer_nonlinearity_BOT.py validator_nonlinearity_BOT.py diff --git a/harnessed_jobs/nonlinearity_BOT/v0/nonlinearity_jh_task.py b/harnessed_jobs/nonlinearity_BOT/v0/nonlinearity_jh_task.py new file mode 100755 index 0000000..154e67b --- /dev/null +++ b/harnessed_jobs/nonlinearity_BOT/v0/nonlinearity_jh_task.py @@ -0,0 +1,29 @@ +#!/usr/bin/env ipython +""" +Script for BOT nonlinearity analysis. +""" +def nonlinearity_jh_task(det_name): + """JH version of single sensor execution of the non-linearity task.""" + import glob + import siteUtils + from bot_eo_analyses import make_file_prefix, nonlinearity_task + + run = siteUtils.getRunNumber() + file_prefix = make_file_prefix(run, det_name) + + try: + detresp_file \ + = siteUtils.dependency_glob(f'{det_name}*_det_response.fits', + jobname='flat_pairs_BOT')[0] + except IndexError: + print("nonlinearity_task: detector response file not found for", + det_name) + return None + + outfile = f'{file_prefix}_nlc.fits' + return nonlinearity_task(run, det_name, detresp_file, outfile) + +if __name__ == '__main__': + import sys + det_name = sys.argv[1] + nonlinearity_jh_task(det_name) diff --git a/harnessed_jobs/nonlinearity_BOT/v0/producer_nonlinearity_BOT.py b/harnessed_jobs/nonlinearity_BOT/v0/producer_nonlinearity_BOT.py new file mode 100755 index 0000000..f0b8da3 --- /dev/null +++ b/harnessed_jobs/nonlinearity_BOT/v0/producer_nonlinearity_BOT.py @@ -0,0 +1,13 @@ +#!/usr/bin/env ipython +""" +Producer script for BOT nonlinearity analysis. +""" +import os +from nonlinearity_jh_task import nonlinearity_jh_task +from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script + +if 'nonlinearity' in get_analysis_types(): + nonlinearity_script \ + = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', + 'nonlinearity_BOT', 'v0', 'nonlinearity_jh_task.py') + run_python_task_or_cl_script(nonlinearity_jh_task, nonlinearity_script) diff --git a/harnessed_jobs/nonlinearity_BOT/v0/validator_nonlinearity_BOT.py b/harnessed_jobs/nonlinearity_BOT/v0/validator_nonlinearity_BOT.py new file mode 100755 index 0000000..e7506f8 --- /dev/null +++ b/harnessed_jobs/nonlinearity_BOT/v0/validator_nonlinearity_BOT.py @@ -0,0 +1,6 @@ +#!/usr/bin/env ipython +""" +Validator script for BOT flat pairs (linearity and full-well) analysis. +""" +from bot_eo_validators import run_validator +run_validator('nonlinearity') diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index 69052d6..d8025f4 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -49,12 +49,14 @@ 'plot_cte_results', 'find_flat2_bot', 'flat_pairs_task', + 'nonlinearity_task', 'ptc_task', 'bf_task', 'qe_task', 'tearing_task', 'repackage_summary_files', 'mondiode_value', + 'get_nlc_func', 'run_jh_tasks', 'run_python_task_or_cl_script'] @@ -145,7 +147,7 @@ def get_mask_files(det_name): rolloff_mask_files = hj_fp_server.get_files('bias_frame_BOT', f'{det_name}_*mask*.fits', run=bias_run) - print(f"Edge rollooff mask file from run {bias_run} and {det_name}:") + print(f"Edge rolloff mask file from run {bias_run} and {det_name}:") for item in rolloff_mask_files: print(item) print() @@ -247,7 +249,8 @@ def fe55_task(run, det_name, fe55_files): task = sensorTest.Fe55Task() task.config.temp_set_point = -100. task.run(file_prefix, fe55_files, (rolloff_mask_file,), - bias_frame=bias_frame, accuracy_req=0.01, hist_nsig=hist_nsig) + bias_frame=bias_frame, accuracy_req=0.01, hist_nsig=hist_nsig, + linearity_correction=get_nlc_func(det_name)) # Fe55 gain and psf analysis results plots for the test report. results_file = '%s_eotest_results.fits' % file_prefix @@ -523,7 +526,8 @@ def bright_defects_task(run, det_name, dark_files, gains, mask_files=(), task = sensorTest.BrightPixelsTask() task.config.temp_set_point = -100. - task.run(file_prefix, dark_files, mask_files, gains, bias_frame=bias_frame) + task.run(file_prefix, dark_files, mask_files, gains, bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) title = '%s, medianed dark for bright defects analysis' % file_prefix annotation = 'e-/pixel, gain-corrected, bias-subtracted' @@ -541,7 +545,8 @@ def dark_defects_task(run, det_name, sflat_files, mask_files=(), title = '{}, {}'.format(run, det_name) task = sensorTest.DarkPixelsTask() - task.run(file_prefix, sflat_files, mask_files, bias_frame=bias_frame) + task.run(file_prefix, sflat_files, mask_files, bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) title = '%s, superflat for dark defects analysis' % file_prefix siteUtils.make_png_file(sensorTest.plot_flat, @@ -554,7 +559,8 @@ def traps_task(run, det_name, trap_file, gains, mask_files=(), bias_frame=None): """Single sensor execution of the traps analysis task.""" file_prefix = make_file_prefix(run, det_name) task = sensorTest.TrapTask() - task.run(file_prefix, trap_file, mask_files, gains, bias_frame=bias_frame) + task.run(file_prefix, trap_file, mask_files, gains, bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) def dark_current_task(run, det_name, dark_files, gains, mask_files=(), @@ -564,7 +570,8 @@ def dark_current_task(run, det_name, dark_files, gains, mask_files=(), task = sensorTest.DarkCurrentTask() task.config.temp_set_point = temp_set_point return task.run(file_prefix, dark_files, mask_files, gains, - bias_frame=bias_frame) + bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) def plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, @@ -587,7 +594,8 @@ def cte_task(run, det_name, sflat_files, gains, mask_files=(), task = sensorTest.CteTask() task.run(file_prefix, sflat_files, flux_level=flux_level, gains=gains, - mask_files=mask_files, bias_frame=bias_frame) + mask_files=mask_files, bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) # TODO: refactor CteTask to provide access to the superflat filename # instead of recomputing it here. superflat_file = '{}_superflat_{}.fits'.format(file_prefix, flux_level) @@ -653,6 +661,7 @@ def flat_pairs_task(run, det_name, flat_files, gains, mask_files=(), linearity_spec_range=linearity_spec_range, use_exptime=use_exptime, flat2_finder=flat2_finder, bias_frame=bias_frame, mondiode_func=mondiode_func, + linearity_correction=get_nlc_func(det_name), dark_frame=dark_frame) results_file = '%s_eotest_results.fits' % file_prefix @@ -671,6 +680,16 @@ def flat_pairs_task(run, det_name, flat_files, gains, mask_files=(), use_exptime=use_exptime) +def nonlinearity_task(run, det_name, detresp_file, outfile): + """Single sensor execution of nonlinearity task.""" + file_prefix = make_file_prefix(run, det_name) + task = sensorTest.NonlinearityTask() + try: + task.run(file_prefix, detresp_file, outputfile=outfile) + except Exception: + print(f'NonlinearityTask.run failed for {file_prefix}') + + def ptc_task(run, det_name, flat_files, gains, mask_files=(), flat2_finder=find_flat2_bot, bias_frame=None): """Single sensor execution of the PTC task.""" @@ -678,7 +697,8 @@ def ptc_task(run, det_name, flat_files, gains, mask_files=(), task = sensorTest.PtcTask() task.run(file_prefix, flat_files, mask_files, gains, - flat2_finder=flat2_finder, bias_frame=bias_frame) + flat2_finder=flat2_finder, bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) @@ -694,7 +714,8 @@ def bf_task(run, det_name, flat_files, mask_files=(), task = sensorTest.BFTask() task.run(file_prefix, flat_files, mask_files=mask_files, - flat2_finder=flat2_finder, bias_frame=bias_frame) + flat2_finder=flat2_finder, bias_frame=bias_frame, + linearity_correction=get_nlc_func(det_name)) results_file = '%s_eotest_results.fits' % file_prefix plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file) @@ -830,6 +851,28 @@ def get_analysis_types(bot_eo_config_file=None): return analysis_types +def get_nlc_func(det_name): + """ + Return the nonlinearity correction function for the specified + detector. + """ + file_prefix = make_file_prefix('*', det_name) + try: + nlc_file = siteUtils.dependency_glob(f'{file_prefix}_nlc.fits', + jobname='nonlinearity_BOT')[0] + except IndexError as eobj: + print(f'{file_prefix}_nlc.fits not found:', eobj) + return None + + try: + nlc = sensorTest.NonlinearityCorrection.create_from_fits_file(nlc_file) + except ValueError as eobj: + print(f'Error creating nlc from {nlc_file}:', eobj) + nlc = None + + return nlc + + def run_jh_tasks(*jh_tasks, device_names=None, processes=None, walltime=3600): """ Run functions to execute tasks under the job harness in parallel. @@ -887,9 +930,10 @@ def run_jh_tasks(*jh_tasks, device_names=None, processes=None, walltime=3600): # Query for file paths for other analysis runs, if specified in # the bot_eo_config_file. - hj_fp_server.query_file_paths(siteUtils.get_analysis_run('badpixel')) - hj_fp_server.query_file_paths(siteUtils.get_analysis_run('bias')) - hj_fp_server.query_file_paths(siteUtils.get_analysis_run('dark')) + for analysis_type in ('badpixel', 'bias', 'dark', 'linearity', + 'nonlinearity'): + hj_fp_server.query_file_paths( + siteUtils.get_analysis_run(analysis_type)) hj_fp_server_file = 'hj_fp_server.pkl' with open(hj_fp_server_file, 'wb') as output: diff --git a/python/bot_eo_validators.py b/python/bot_eo_validators.py index e9c2562..f3995da 100644 --- a/python/bot_eo_validators.py +++ b/python/bot_eo_validators.py @@ -25,7 +25,7 @@ 'validate_cte', 'validate_flat_pairs', 'validate_ptc', 'validate_brighter_fatter', 'validate_qe', 'validate_tearing', 'validate_raft_results', - 'validate_flat_gain_stability'] + 'validate_flat_gain_stability', 'validate_nonlinearity'] def run_validator(*det_task_names): @@ -772,3 +772,19 @@ def validate_raft_results(results, raft_names): components='rafts', total=21) return results + + +def validate_nonlinearity(results, det_names): + """Validate the nonlinearity analysis results.""" + run = siteUtils.getRunNumber() + results = [] + missing_det_names = [] + for det_name in det_names: + nlc_file = f'{make_file_prefix(run, det_name)}_nlc.fits' + if not os.path.isfile(nlc_file): + missing_det_names.append(det_name) + continue + md = dict(DATA_PRODUCT='nonlinearity_correction') + results.append(siteUtils.make_fileref(nlc_file, metadata=md)) + report_missing_data('validate_nonlinearity', missing_det_names) + return results