Merge pull request #57 from lsst-camera-dh/LSSTTD-1470_add_overscan_task

Lssttd 1470 add overscan task

data/BOT_jh_glob_patterns.ini

@@ -12,6 +12,7 @@ cte_high = sflat_flat_*H*
 cte_low = sflat_flat_*L*
 flat_pairs = flat_*_flat?_*
 ptc = flat_*_flat?_*
+overscan = flat_*_flat?_*
 brighter_fatter = flat_*_flat?_*
 qe = lambda_flat_*
 tearing = sflat_flat_*[LH]*

harnessed_jobs/overscan_BOT/v0/modulefile

@@ -0,0 +1,3 @@
+#%Module1.0 #-*-tcl-*-#
+source "$::env(LCATR_MODULES)/lcatr.tcl"
+lcatr_package producer_overscan_BOT.py validator_overscan_BOT.py

harnessed_jobs/overscan_BOT/v0/overscan_jh_task.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env ipython
+"""
+Harnessed job script for BOT overscan analysis.
+"""
+def overscan_jh_task(det_name):
+    """JH version of single sensor execution of the overscan task."""
+    import siteUtils
+    from bot_eo_analyses import make_file_prefix, glob_pattern,\
+        get_amplifier_gains, bias_filename, overscan_task
+
+    run = siteUtils.getRunNumber()
+    file_prefix = make_file_prefix(run, det_name)
+    acq_jobname = siteUtils.getProcessName('BOT_acq')
+
+    flat_files = siteUtils.dependency_glob(glob_pattern('overscan', det_name),
+                                           acq_jobname=acq_jobname)
+    if not flat_files:
+        print("overscan_task: Flat pairs files not found for detector",
+              det_name)
+        return None
+
+    eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
+    gains = get_amplifier_gains(eotest_results_file)
+    bias_frame = bias_filename(run, det_name)
+
+    return overscan_task(run, det_name, flat_files, gains,
+                         bias_frame=bias_frame)
+
+if __name__ == '__main__':
+    import sys
+    det_name = sys.argv[1]
+    overscan_jh_task(det_name)

harnessed_jobs/overscan_BOT/v0/producer_overscan_BOT.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env ipython
+"""
+Producer script for BOT overscan analysis.
+"""
+import os
+from overscan_jh_task import overscan_jh_task
+from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script
+
+if 'overscan' in get_analysis_types():
+    overscan_task_script \
+        = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs',
+                       'overscan_BOT', 'v0', 'overscan_jh_task.py')
+
+    run_python_task_or_cl_script(overscan_jh_task, overscan_task_script)

harnessed_jobs/overscan_BOT/v0/validator_overscan_BOT.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env ipython
+"""
+Validator script for BOT overscan analysis.
+"""
+from bot_eo_validators import run_validator
+run_validator('overscan')

python/bot_eo_analyses.py

@@ -17,6 +17,7 @@
 from astropy.io import fits
 import lsst.eotest.image_utils as imutils
 import lsst.eotest.sensor as sensorTest
+from lsst.eotest.sensor.EOTestPlots import OverscanTestPlots
 import eotestUtils
 import siteUtils
 from correlated_noise import correlated_noise, raft_level_oscan_correlations
@@ -802,6 +803,27 @@ def tearing_task(run, det_name, flat_files, bias_frame=None):
         pickle.dump(tearing_stats, output)
 
 
+def overscan_task(run, det_name, flat_files, gains, bias_frame=None):
+    """Single sensor execution of the overscan task."""
+    file_prefix = make_file_prefix(run, det_name)
+    task = sensorTest.OverscanTask()
+    task.run(file_prefix, flat_files, gains, bias_frame=bias_frame)
+
+    overscan_file = f'{file_prefix}_overscan_results.fits'
+    plots = OverscanTestPlots(file_prefix, overscan_file=overscan_file)
+
+    plot_types = '''serial_eper_low serial_eper_high serial_cti
+                    serial_overscan_signal serial_overscan_sum
+                    parallel_eper_low parallel_eper_high parallel_cti
+                    parallel_overscan_signal parallel_overscan_sum'''.split()
+
+    plot_funcs = {_: getattr(plots, f'{_}_curves') for _ in plot_types}
+
+    for plot_type, plot_func in plot_funcs.items():
+        siteUtils.make_png_file(plot_func, f'{file_prefix}_{plot_type}.png')
+        plt.close()
+
+
 def get_raft_files_by_slot(raft_name, file_suffix, jobname=None):
     """Return a dictionary of raft filenames, keyed by slot_name."""
     run = '*'

python/bot_eo_validators.py

@@ -25,7 +25,8 @@
            'validate_cte', 'validate_flat_pairs', 'validate_ptc',
            'validate_brighter_fatter',
            'validate_qe', 'validate_tearing', 'validate_raft_results',
-           'validate_flat_gain_stability', 'validate_nonlinearity']
+           'validate_flat_gain_stability', 'validate_nonlinearity',
+           'validate_overscan']
 
 
 def run_validator(*det_task_names):
@@ -788,3 +789,27 @@ def validate_nonlinearity(results, det_names):
         results.append(siteUtils.make_fileref(nlc_file, metadata=md))
     report_missing_data('validate_nonlinearity', missing_det_names)
     return results
+
+
+def validate_overscan(results, det_names):
+    """Validate the overscan analysis results."""
+    run = siteUtils.getRunNumber()
+    results = []
+    missing_det_names = []
+    for det_name in det_names:
+        file_prefix = make_file_prefix(run, det_name)
+        results_file = f'{file_prefix}_overscan_results.fits'
+        if not os.path.isfile(results_file):
+            missing_det_names.append(det_name)
+        else:
+            md = dict(DATA_PRODUCT='overscan_task_results', RUN=run,
+                      DETECTOR=det_name)
+            results.append(siteUtils.make_fileref(results_file, metadata=md))
+        png_files = (glob.glob(f'{file_prefix}_*_eper_*.png')
+                     + glob.glob(f'{file_prefix}_*_overscan_*.png'))
+        md = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
+        results.extend(siteUtils.persist_png_files('', file_prefix,
+                                                   png_files=png_files,
+                                                   metadata=md))
+    report_missing_data('validate_overscan', missing_det_names)
+    return results