Merge pull request #51 from lsst-camera-dh/LSSTTD-1447_add_gain_stabi…

…lity_analysis

Lssttd 1447 add gain stability analysis

harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py

@@ -14,7 +14,8 @@ def bias_frame_jh_task(det_name):
     nbias = os.environ.get('LCATR_NUM_BIAS_FRAMES', 10)
     bias_files \
         = siteUtils.dependency_glob(glob_pattern('bias_frame', det_name),
-                                    acq_jobname=acq_jobname)[:nbias]
+                                    acq_jobname=acq_jobname,
+                                    description='Bias frames:')[:nbias]
     if not bias_files:
         print("bias_frame_task: Needed data files are missing for detector",
               det_name)

harnessed_jobs/brighter_fatter_BOT/v0/bf_jh_task.py

@@ -7,7 +7,7 @@ def bf_jh_task(det_name):
     import glob
     import siteUtils
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
-        bias_filename, bf_task, find_flat2_bot
+        bias_filename, bf_task, find_flat2_bot, get_mask_files
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
@@ -23,9 +23,9 @@ def bf_jh_task(det_name):
 
     flat_files = [_ for _ in flat_files if 'flat1' in _]
 
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
+    mask_files = get_mask_files(det_name)
     eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
-    bias_frame = bias_filename(file_prefix)
+    bias_frame = bias_filename(run, det_name)
 
     return bf_task(run, det_name, flat_files, mask_files=mask_files,
                    flat2_finder=find_flat2_bot, bias_frame=bias_frame)

harnessed_jobs/cti_BOT/v0/cte_jh_task.py

@@ -8,8 +8,10 @@ def cte_jh_task(det_name):
     import glob
     import shutil
     import siteUtils
+    import lsst.eotest.sensor as sensorTest
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
-        get_amplifier_gains, bias_filename, cte_task, plot_cte_results
+        get_amplifier_gains, bias_filename, cte_task, plot_cte_results,\
+        get_mask_files
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
@@ -25,19 +27,19 @@ def cte_jh_task(det_name):
         print("cte_task: Superflat files not found for detector", det_name)
         return None
 
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
+    mask_files = get_mask_files(det_name)
 
     eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
     gains = get_amplifier_gains(eotest_results_file)
-    try:
-        results_file = siteUtils.dependency_glob(eotest_results_file,
-                                                 jobname='fe55_analysis_BOT')[0]
-    except IndexError:
-        pass
-    else:
-        shutil.copy(results_file, eotest_results_file)
 
-    bias_frame = bias_filename(file_prefix)
+    # Write gains to local eotest_results_file, which cte_task will update.
+    namps = 16 if 'SW' not in det_name else 8
+    results = sensorTest.EOTestResults(eotest_results_file, namps=namps)
+    for amp, gain in gains.items():
+        results.add_seg_result(amp, 'GAIN', gain)
+    results.write()
+
+    bias_frame = bias_filename(run, det_name)
 
     # Omit rolloff defects mask since it would mask some of the edges used
     # in the eper method.

harnessed_jobs/dark_current_BOT/v0/dark_current_jh_task.py

@@ -8,25 +8,26 @@ def dark_current_jh_task(det_name):
     import siteUtils
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
         get_amplifier_gains, bias_filename, dark_current_task,\
-        plot_ccd_total_noise
+        plot_ccd_total_noise, get_mask_files
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
     acq_jobname = siteUtils.getProcessName('BOT_acq')
 
     dark_files \
         = siteUtils.dependency_glob(glob_pattern('dark_current', det_name),
-                                    acq_jobname=acq_jobname)
+                                    acq_jobname=acq_jobname,
+                                    description="Dark current frames:")
     if not dark_files:
         print("dark_current_task: No dark files found for detector", det_name)
         return None
 
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
+    mask_files = get_mask_files(det_name)
     eotest_results_file \
         = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix),
                                     jobname='read_noise_BOT')[0]
     gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix))
-    bias_frame = bias_filename(file_prefix)
+    bias_frame = bias_filename(run, det_name)
 
     dark_curr_pixels, dark95s \
         = dark_current_task(run, det_name, dark_files, gains,

harnessed_jobs/fe55_analysis_BOT/v0/gain_stability_jh_task.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+"""
+Producer script for BOT gain stability analysis.  The fe55_jh_task.py
+script needs to be executed before this script.
+"""
+def gain_stability_jh_task(det_name):
+    "JH version of the gain stability analysis task."
+    import os
+    import siteUtils
+    from bot_eo_analyses import glob_pattern, gain_stability_task
+
+    run = siteUtils.getRunNumber()
+    acq_jobname = siteUtils.getProcessName('BOT_acq')
+
+    fe55_files = siteUtils.dependency_glob(glob_pattern('fe55', det_name),
+                                           acq_jobname=acq_jobname)
+    if not fe55_files:
+        print("fe55_task: Needed data files missing for detector", det_name)
+        return None
+    return gain_stability_task(run, det_name, fe55_files)
+
+if __name__ == '__main__':
+    import sys
+    det_name = sys.argv[1]
+    gain_stability_jh_task(det_name)

harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py

@@ -4,11 +4,18 @@
 """
 import os
 from fe55_jh_task import fe55_jh_task
+from gain_stability_jh_task import gain_stability_jh_task
 from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script
 
-if 'gain' in get_analysis_types():
-    fe55_task_script \
-        = os.path.join(os.environ['EOANALYSISJOBSDIR'],
-                       'harnessed_jobs', 'fe55_analysis_BOT',
-                       'v0', 'fe55_jh_task.py')
+job_dir = os.path.join(os.environ['EOANALYSISJOBSDIR'],
+                       'harnessed_jobs', 'fe55_analysis_BOT', 'v0')
+
+analysis_types = get_analysis_types()
+
+if 'gain' in analysis_types or 'gainstability' in analysis_types:
+    fe55_task_script = os.path.join(job_dir, 'fe55_jh_task.py')
     run_python_task_or_cl_script(fe55_jh_task, fe55_task_script)
+
+if 'gainstability' in analysis_types:
+    gain_stability_script = os.path.join(job_dir, 'gain_stability_jh_task.py')
+    run_python_task_or_cl_script(gain_stability_jh_task, gain_stability_script)

harnessed_jobs/flat_gain_stability_BOT/v0/flat_gain_stability_jh_task.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Script for BOT flat gain stability analysis.
+"""
+def flat_gain_stability_jh_task(det_name):
+    """JH version of single sensor execution of the flat pairs task."""
+    import glob
+    import siteUtils
+    from bot_eo_analyses import make_file_prefix, glob_pattern,\
+        bias_filename, flat_gain_stability_task,\
+        get_mask_files, medianed_dark_frame
+
+    run = siteUtils.getRunNumber()
+    file_prefix = make_file_prefix(run, det_name)
+    acq_jobname = siteUtils.getProcessName('BOT_acq')
+
+    flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name),
+                                           acq_jobname=acq_jobname)
+    if not flat_files:
+        print("flat_gain_stability_task: Flat pairs files not found for",
+              det_name)
+        return None
+
+    mask_files = get_mask_files(det_name)
+    bias_frame = bias_filename(run, det_name)
+    dark_frame = medianed_dark_frame(det_name)
+
+    return flat_gain_stability_task(run, det_name, flat_files,
+                                    mask_files=mask_files,
+                                    bias_frame=bias_frame,
+                                    dark_frame=dark_frame)
+
+
+if __name__ == '__main__':
+    import sys
+    det_name = sys.argv[1]
+    flat_gain_stability_jh_task(det_name)

harnessed_jobs/flat_gain_stability_BOT/v0/modulefile

@@ -0,0 +1,3 @@
+#%Module1.0 #-*-tcl-*-#
+source "$::env(LCATR_MODULES)/lcatr.tcl"
+lcatr_package producer_flat_gain_stability_BOT.py validator_flat_gain_stability_BOT.py

harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env ipython
+"""
+Producer script for BOT flat gain stability analysis.
+"""
+import os
+import siteUtils
+from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script
+from flat_gain_stability import plot_all_rafts
+from flat_gain_stability_jh_task import flat_gain_stability_jh_task
+
+if 'gainstability' in get_analysis_types():
+    flat_gain_stability_script \
+        = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs',
+                       'flat_gain_stability_BOT', 'v0',
+                       'flat_gain_stability_jh_task.py')
+    run_python_task_or_cl_script(flat_gain_stability_jh_task,
+                                 flat_gain_stability_script)
+
+    plot_all_rafts(siteUtils.getRunNumber())

harnessed_jobs/flat_gain_stability_BOT/v0/validator_flat_gain_stability_BOT.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+"""
+Validator script for BOT flat pairs (linearity and full-well) analysis.
+"""
+from bot_eo_validators import run_validator
+run_validator('flat_gain_stability')

harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py

@@ -7,7 +7,8 @@ def flat_pairs_jh_task(det_name):
     import glob
     import siteUtils
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
-        get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value
+        get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value,\
+        get_mask_files, medianed_dark_frame
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
@@ -21,14 +22,15 @@ def flat_pairs_jh_task(det_name):
               det_name)
         return None
 
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
+    mask_files = get_mask_files(det_name)
     eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
     gains = get_amplifier_gains(eotest_results_file)
-    bias_frame = bias_filename(file_prefix)
+    bias_frame = bias_filename(run, det_name)
+    dark_frame = medianed_dark_frame(det_name)
 
     return flat_pairs_task(run, det_name, flat_files, gains,
                            mask_files=mask_files, bias_frame=bias_frame,
-                           mondiode_func=mondiode_value)
+                           mondiode_func=mondiode_value, dark_frame=dark_frame)
 
 
 if __name__ == '__main__':

harnessed_jobs/pixel_defects_BOT/v0/bright_defects_jh_task.py

@@ -7,7 +7,7 @@ def bright_defects_jh_task(det_name):
     import glob
     import siteUtils
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
-        get_amplifier_gains, bias_filename, bright_defects_task
+        get_amplifier_gains, bias_filename, bright_defects_task, get_mask_files
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
@@ -23,9 +23,8 @@ def bright_defects_jh_task(det_name):
 
     eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
     gains = get_amplifier_gains(eotest_results_file)
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
-
-    bias_frame = bias_filename(file_prefix)
+    mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits'))
+    bias_frame = bias_filename(run, det_name)
 
     return bright_defects_task(run, det_name, dark_files, gains,
                                mask_files=mask_files, bias_frame=bias_frame)

harnessed_jobs/pixel_defects_BOT/v0/dark_defects_jh_task.py

@@ -7,7 +7,7 @@ def dark_defects_jh_task(det_name):
     import glob
     import siteUtils
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
-        get_amplifier_gains, bias_filename, dark_defects_task
+        get_amplifier_gains, bias_filename, dark_defects_task, get_mask_files
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
@@ -21,8 +21,8 @@ def dark_defects_jh_task(det_name):
               det_name)
         return None
 
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
-    bias_frame = bias_filename(file_prefix)
+    mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits'))
+    bias_frame = bias_filename(run, det_name)
 
     return dark_defects_task(run, det_name, sflat_files, mask_files=mask_files,
                              bias_frame=bias_frame)

harnessed_jobs/ptc_BOT/v0/ptc_jh_task.py

@@ -7,7 +7,7 @@ def ptc_jh_task(det_name):
     import glob
     import siteUtils
     from bot_eo_analyses import make_file_prefix, glob_pattern,\
-        get_amplifier_gains, bias_filename, ptc_task
+        get_amplifier_gains, bias_filename, ptc_task, get_mask_files
 
     run = siteUtils.getRunNumber()
     file_prefix = make_file_prefix(run, det_name)
@@ -19,10 +19,10 @@ def ptc_jh_task(det_name):
         print("ptc_task: Flat pairs files not found for detector", det_name)
         return None
 
-    mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
+    mask_files = get_mask_files(det_name)
     eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
     gains = get_amplifier_gains(eotest_results_file)
-    bias_frame = bias_filename(file_prefix)
+    bias_frame = bias_filename(run, det_name)
 
     return ptc_task(run, det_name, flat_files, gains,
                     mask_files=mask_files, bias_frame=bias_frame)

harnessed_jobs/raft_results_summary_BOT/v0/producer_raft_results_summary_BOT.py

@@ -17,32 +17,43 @@ def make_focal_plane_plots():
     #
     fp_configs = {'fe55_BOT_analysis': (('gain', 'psf_sigma'),
                                         ((0.5, 1.5), (3, 5.5)),
-                                        (('e-/ADU', 'micron'))),
-                  'read_noise_BOT': (('read_noise',), ((0, 20),), ('e-',)),
+                                        ('e-/ADU', 'micron'),
+                                        (False, False),
+                                        ('1', '1')),
+                  'read_noise_BOT': (('read_noise',), ((0, 20),), ('e-',),
+                                     (False,), ('1',)),
                   'bright_defects_BOT': (('bright_pixels', 'bright_columns'),
-                                         (None, None), (None, None)),
+                                         (None, None), (None, None),
+                                         (True, True), ('1', '1')),
                   'dark_defects_BOT': (('dark_pixels', 'dark_columns'),
-                                       (None, None), (None, None)),
+                                       (None, None), (None, None),
+                                       (True, True), ('1', '1')),
                   'dark_current_BOT': (('dark_current_95CL',), ((0, 0.3),),
-                                       ('e-/pix/s',)),
-                  'traps_BOT': (('num_traps',), (None,), (None,)),
+                                       ('e-/pix/s',), (False,), ('1',)),
+                  'traps_BOT': (('num_traps',), (None,), (None,), (False,),
+                                ('1',)),
                   'cte_BOT': (('cti_high_serial', 'cti_high_parallel',
                                'cti_low_serial', 'cti_low_parallel'),
                               ((0, 1e-5), (0, 1e-5), (0, 1e-5), (0, 1e-5)),
-                              (None, None, None, None)),
-                  'flat_pairs_BOT': (('max_frac_dev',), ((0, 0.05),), (None,)),
+                              (None, None, None, None),
+                              (False, False, False, False),
+                              ('1e-6', '1e-6', '1e-6', '1e-6')),
+                  'flat_pairs_BOT': (('max_frac_dev', 'full_well'),
+                                     ((0, 0.05), (0, 2e5)), (None, None),
+                                     (False, False), ('1', '1')),
                   'ptc_BOT': (('ptc_gain', 'ptc_a00'), ((0.5, 1.5), (0, 5e-6)),
-                              ('e-/ADU', None)),
+                              ('e-/ADU', None), (False, False), ('1', '1')),
                   'bright_fatter_BOT': (('bf_xcorr', 'bf_ycorr'), (None, None),
-                                        (None, None))}
+                                        (None, None), (False, False),
+                                        ('1', '1'))}
 
     camera = camera_info.camera_object
 
     run = siteUtils.getRunNumber()
     unit_id = siteUtils.getUnitId()
     et_results = siteUtils.ETResults(run)
     for schema_name, configs in fp_configs.items():
-        for column, z_range, units in zip(*configs):
+        for column, z_range, units, use_log10, scale_factor in zip(*configs):
             fig = plt.figure(figsize=(12, 10))
             ax = fig.add_subplot(1, 1, 1)
             try:
@@ -51,7 +62,9 @@ def make_focal_plane_plots():
                 print("No focal plane results for {}: {} in eTraveler."
                       .format(schema_name, column))
             else:
-                plot_focal_plane(ax, amp_data, camera=camera, z_range=z_range)
+                plot_focal_plane(ax, amp_data, camera=camera, z_range=z_range,
+                                 use_log10=use_log10,
+                                 scale_factor=scale_factor)
                 if units is not None:
                     plt.title('Run {}, {} ({})'.format(run, column, units))
                 else:
@@ -61,7 +74,8 @@ def make_focal_plane_plots():
 
                 # Histogram the amp-level data.
                 plt.figure()
-                hist_amp_data(amp_data, column, hist_range=z_range)
+                hist_amp_data(amp_data, column, hist_range=z_range,
+                              use_log10=use_log10, scale_factor=scale_factor)
                 if units is not None:
                     plt.title('Run {}, {} ({})'.format(run, column, units))
                 else: