Merge pull request #54 from lsst-camera-dh/u/jchiang/BOT_nonlinearity…

…_job

add BOT nonlinearity job

harnessed_jobs/nonlinearity_BOT/v0/modulefile

@@ -0,0 +1,3 @@
+#%Module1.0 #-*-tcl-*-#
+source "$::env(LCATR_MODULES)/lcatr.tcl"
+lcatr_package producer_nonlinearity_BOT.py validator_nonlinearity_BOT.py

harnessed_jobs/nonlinearity_BOT/v0/nonlinearity_jh_task.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env ipython
+"""
+Script for BOT nonlinearity analysis.
+"""
+def nonlinearity_jh_task(det_name):
+    """JH version of single sensor execution of the non-linearity task."""
+    import glob
+    import siteUtils
+    from bot_eo_analyses import make_file_prefix, nonlinearity_task
+
+    run = siteUtils.getRunNumber()
+    file_prefix = make_file_prefix(run, det_name)
+
+    try:
+        detresp_file \
+            = siteUtils.dependency_glob(f'{det_name}*_det_response.fits',
+                                        jobname='flat_pairs_BOT')[0]
+    except IndexError:
+        print("nonlinearity_task: detector response file not found for",
+              det_name)
+        return None
+
+    outfile = f'{file_prefix}_nlc.fits'
+    return nonlinearity_task(run, det_name, detresp_file, outfile)
+
+if __name__ == '__main__':
+    import sys
+    det_name = sys.argv[1]
+    nonlinearity_jh_task(det_name)

harnessed_jobs/nonlinearity_BOT/v0/producer_nonlinearity_BOT.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env ipython
+"""
+Producer script for BOT nonlinearity analysis.
+"""
+import os
+from nonlinearity_jh_task import nonlinearity_jh_task
+from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script
+
+if 'nonlinearity' in get_analysis_types():
+    nonlinearity_script \
+        = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs',
+                    'nonlinearity_BOT', 'v0', 'nonlinearity_jh_task.py')
+    run_python_task_or_cl_script(nonlinearity_jh_task, nonlinearity_script)

harnessed_jobs/nonlinearity_BOT/v0/validator_nonlinearity_BOT.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env ipython
+"""
+Validator script for BOT flat pairs (linearity and full-well) analysis.
+"""
+from bot_eo_validators import run_validator
+run_validator('nonlinearity')

python/bot_eo_analyses.py

@@ -49,12 +49,14 @@
            'plot_cte_results',
            'find_flat2_bot',
            'flat_pairs_task',
+           'nonlinearity_task',
            'ptc_task',
            'bf_task',
            'qe_task',
            'tearing_task',
            'repackage_summary_files',
            'mondiode_value',
+           'get_nlc_func',
            'run_jh_tasks',
            'run_python_task_or_cl_script']
 
@@ -145,7 +147,7 @@ def get_mask_files(det_name):
         rolloff_mask_files = hj_fp_server.get_files('bias_frame_BOT',
                                                     f'{det_name}_*mask*.fits',
                                                     run=bias_run)
-        print(f"Edge rollooff mask file from run {bias_run} and {det_name}:")
+        print(f"Edge rolloff mask file from run {bias_run} and {det_name}:")
         for item in rolloff_mask_files:
             print(item)
         print()
@@ -247,7 +249,8 @@ def fe55_task(run, det_name, fe55_files):
     task = sensorTest.Fe55Task()
     task.config.temp_set_point = -100.
     task.run(file_prefix, fe55_files, (rolloff_mask_file,),
-             bias_frame=bias_frame, accuracy_req=0.01, hist_nsig=hist_nsig)
+             bias_frame=bias_frame, accuracy_req=0.01, hist_nsig=hist_nsig,
+             linearity_correction=get_nlc_func(det_name))
 
     # Fe55 gain and psf analysis results plots for the test report.
     results_file = '%s_eotest_results.fits' % file_prefix
@@ -523,7 +526,8 @@ def bright_defects_task(run, det_name, dark_files, gains, mask_files=(),
 
     task = sensorTest.BrightPixelsTask()
     task.config.temp_set_point = -100.
-    task.run(file_prefix, dark_files, mask_files, gains, bias_frame=bias_frame)
+    task.run(file_prefix, dark_files, mask_files, gains, bias_frame=bias_frame,
+             linearity_correction=get_nlc_func(det_name))
 
     title = '%s, medianed dark for bright defects analysis' % file_prefix
     annotation = 'e-/pixel, gain-corrected, bias-subtracted'
@@ -541,7 +545,8 @@ def dark_defects_task(run, det_name, sflat_files, mask_files=(),
     title = '{}, {}'.format(run, det_name)
 
     task = sensorTest.DarkPixelsTask()
-    task.run(file_prefix, sflat_files, mask_files, bias_frame=bias_frame)
+    task.run(file_prefix, sflat_files, mask_files, bias_frame=bias_frame,
+             linearity_correction=get_nlc_func(det_name))
 
     title = '%s, superflat for dark defects analysis' % file_prefix
     siteUtils.make_png_file(sensorTest.plot_flat,
@@ -554,7 +559,8 @@ def traps_task(run, det_name, trap_file, gains, mask_files=(), bias_frame=None):
     """Single sensor execution of the traps analysis task."""
     file_prefix = make_file_prefix(run, det_name)
     task = sensorTest.TrapTask()
-    task.run(file_prefix, trap_file, mask_files, gains, bias_frame=bias_frame)
+    task.run(file_prefix, trap_file, mask_files, gains, bias_frame=bias_frame,
+             linearity_correction=get_nlc_func(det_name))
 
 
 def dark_current_task(run, det_name, dark_files, gains, mask_files=(),
@@ -564,7 +570,8 @@ def dark_current_task(run, det_name, dark_files, gains, mask_files=(),
     task = sensorTest.DarkCurrentTask()
     task.config.temp_set_point = temp_set_point
     return task.run(file_prefix, dark_files, mask_files, gains,
-                    bias_frame=bias_frame)
+                    bias_frame=bias_frame,
+                    linearity_correction=get_nlc_func(det_name))
 
 
 def plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s,
@@ -587,7 +594,8 @@ def cte_task(run, det_name, sflat_files, gains, mask_files=(),
 
     task = sensorTest.CteTask()
     task.run(file_prefix, sflat_files, flux_level=flux_level, gains=gains,
-             mask_files=mask_files, bias_frame=bias_frame)
+             mask_files=mask_files, bias_frame=bias_frame,
+             linearity_correction=get_nlc_func(det_name))
     # TODO: refactor CteTask to provide access to the superflat filename
     # instead of recomputing it here.
     superflat_file = '{}_superflat_{}.fits'.format(file_prefix, flux_level)
@@ -653,6 +661,7 @@ def flat_pairs_task(run, det_name, flat_files, gains, mask_files=(),
              linearity_spec_range=linearity_spec_range,
              use_exptime=use_exptime, flat2_finder=flat2_finder,
              bias_frame=bias_frame, mondiode_func=mondiode_func,
+             linearity_correction=get_nlc_func(det_name),
              dark_frame=dark_frame)
 
     results_file = '%s_eotest_results.fits' % file_prefix
@@ -671,14 +680,25 @@ def flat_pairs_task(run, det_name, flat_files, gains, mask_files=(),
                             use_exptime=use_exptime)
 
 
+def nonlinearity_task(run, det_name, detresp_file, outfile):
+    """Single sensor execution of nonlinearity task."""
+    file_prefix = make_file_prefix(run, det_name)
+    task = sensorTest.NonlinearityTask()
+    try:
+        task.run(file_prefix, detresp_file, outputfile=outfile)
+    except Exception:
+        print(f'NonlinearityTask.run failed for {file_prefix}')
+
+
 def ptc_task(run, det_name, flat_files, gains, mask_files=(),
              flat2_finder=find_flat2_bot, bias_frame=None):
     """Single sensor execution of the PTC task."""
     file_prefix = make_file_prefix(run, det_name)
 
     task = sensorTest.PtcTask()
     task.run(file_prefix, flat_files, mask_files, gains,
-             flat2_finder=flat2_finder, bias_frame=bias_frame)
+             flat2_finder=flat2_finder, bias_frame=bias_frame,
+             linearity_correction=get_nlc_func(det_name))
 
     results_file = '%s_eotest_results.fits' % file_prefix
     plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file)
@@ -694,7 +714,8 @@ def bf_task(run, det_name, flat_files, mask_files=(),
 
     task = sensorTest.BFTask()
     task.run(file_prefix, flat_files, mask_files=mask_files,
-             flat2_finder=flat2_finder, bias_frame=bias_frame)
+             flat2_finder=flat2_finder, bias_frame=bias_frame,
+             linearity_correction=get_nlc_func(det_name))
 
     results_file = '%s_eotest_results.fits' % file_prefix
     plots = sensorTest.EOTestPlots(file_prefix, results_file=results_file)
@@ -830,6 +851,28 @@ def get_analysis_types(bot_eo_config_file=None):
     return analysis_types
 
 
+def get_nlc_func(det_name):
+    """
+    Return the nonlinearity correction function for the specified
+    detector.
+    """
+    file_prefix = make_file_prefix('*', det_name)
+    try:
+        nlc_file = siteUtils.dependency_glob(f'{file_prefix}_nlc.fits',
+                                             jobname='nonlinearity_BOT')[0]
+    except IndexError as eobj:
+        print(f'{file_prefix}_nlc.fits not found:', eobj)
+        return None
+
+    try:
+        nlc = sensorTest.NonlinearityCorrection.create_from_fits_file(nlc_file)
+    except ValueError as eobj:
+        print(f'Error creating nlc from {nlc_file}:', eobj)
+        nlc = None
+
+    return nlc
+
+
 def run_jh_tasks(*jh_tasks, device_names=None, processes=None, walltime=3600):
     """
     Run functions to execute tasks under the job harness in parallel.
@@ -887,9 +930,10 @@ def run_jh_tasks(*jh_tasks, device_names=None, processes=None, walltime=3600):
 
     # Query for file paths for other analysis runs, if specified in
     # the bot_eo_config_file.
-    hj_fp_server.query_file_paths(siteUtils.get_analysis_run('badpixel'))
-    hj_fp_server.query_file_paths(siteUtils.get_analysis_run('bias'))
-    hj_fp_server.query_file_paths(siteUtils.get_analysis_run('dark'))
+    for analysis_type in ('badpixel', 'bias', 'dark', 'linearity',
+                          'nonlinearity'):
+        hj_fp_server.query_file_paths(
+            siteUtils.get_analysis_run(analysis_type))
 
     hj_fp_server_file = 'hj_fp_server.pkl'
     with open(hj_fp_server_file, 'wb') as output:

python/bot_eo_validators.py

@@ -25,7 +25,7 @@
            'validate_cte', 'validate_flat_pairs', 'validate_ptc',
            'validate_brighter_fatter',
            'validate_qe', 'validate_tearing', 'validate_raft_results',
-           'validate_flat_gain_stability']
+           'validate_flat_gain_stability', 'validate_nonlinearity']
 
 
 def run_validator(*det_task_names):
@@ -772,3 +772,19 @@ def validate_raft_results(results, raft_names):
                         components='rafts', total=21)
 
     return results
+
+
+def validate_nonlinearity(results, det_names):
+    """Validate the nonlinearity analysis results."""
+    run = siteUtils.getRunNumber()
+    results = []
+    missing_det_names = []
+    for det_name in det_names:
+        nlc_file = f'{make_file_prefix(run, det_name)}_nlc.fits'
+        if not os.path.isfile(nlc_file):
+            missing_det_names.append(det_name)
+            continue
+        md = dict(DATA_PRODUCT='nonlinearity_correction')
+        results.append(siteUtils.make_fileref(nlc_file, metadata=md))
+    report_missing_data('validate_nonlinearity', missing_det_names)
+    return results