Merge pull request #6 from lsst-camera-dh/LSSTTD-1032

Plotting and analysis requests for TS8

Run successfully in eT Dev:

http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayActivity.jsp?activityId=30075&dataSourceMode=Dev

harnessed_jobs/bright_defects_raft/v0/producer_bright_defects_raft.py

@@ -9,6 +9,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_bright_pixels_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     dark_files = siteUtils.dependency_glob('S*/%s_dark_dark_*.fits' % sensor_id,
                                            jobname=siteUtils.getProcessName('dark_raft_acq'),
                                            description='Dark files:')
@@ -22,7 +23,7 @@ def run_bright_pixels_task(sensor_id):
     task.run(sensor_id, dark_files, mask_files, gains)
 
     siteUtils.make_png_file(sensorTest.plot_flat,
-                            '%s_medianed_dark.png' % sensor_id,
+                            '%s_medianed_dark.png' % file_prefix,
                             '%s_median_dark_bp.fits' % sensor_id,
                             title='%s, medianed dark for bright defects analysis' % sensor_id)
 

harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py

@@ -6,6 +6,7 @@
 from __future__ import print_function
 import os
 from collections import OrderedDict
+import numpy as np
 import matplotlib.pyplot as plt
 import lsst.eotest.sensor as sensorTest
 import lsst.eotest.raft as raftTest
@@ -20,11 +21,17 @@ def slot_dependency_glob(pattern, jobname):
                                              jobname=jobname))
     return OrderedDict([(fn.split('/')[-2], fn) for fn in files])
 
+run_number = siteUtils.getRunNumber()
+
 # Use a mean bias file to determine the maximum number of active
 # pixels for the image quality statistics.
 bias_files = slot_dependency_glob('*_mean_bias_*.fits', 'fe55_raft_analysis')
 total_num, rolloff_mask = sensorTest.pixel_counts(bias_files.values()[0])
 
+# Exposure time (in seconds) for 95th percentile dark current shot
+# noise calculation.
+exptime = 16.
+
 raft_id = siteUtils.getUnitId()
 raft = camera_components.Raft.create_from_etrav(raft_id)
 summary_files = dependency_glob('summary.lims')
@@ -38,6 +45,16 @@ def slot_dependency_glob(pattern, jobname):
                                              rolloff_mask)
     repackager.process_files(summary_files, sensor_id=sensor_id)
 
+    # Add 95th percentile dark current shot noise and add in quadrature
+    # to read noise to produce updated total noise.
+    shot_noise = repackager.eotest_results['DARK_CURRENT_95']*exptime
+    total_noise = np.sqrt(repackager.eotest_results['READ_NOISE']**2
+                          + shot_noise**2)
+    for i, amp in enumerate(repackager.eotest_results['AMP']):
+        repackager.eotest_results.add_seg_result(amp, 'DC95_SHOT_NOISE',
+                                                 np.float(shot_noise[i]))
+        repackager.eotest_results['TOTAL_NOISE'][i] = total_noise[i]
+
     outfile = '%s_eotest_results.fits' % sensor_id
     repackager.write(outfile)
     results_files[slot] = outfile
@@ -48,29 +65,35 @@ def slot_dependency_glob(pattern, jobname):
     gains[slot] = dict([(amp, gain) for amp, gain
                         in zip(results['AMP'], results['GAIN'])])
 
+title = '%s, %s' % (raft_id, run_number)
+file_prefix = '%s_%s' % (raft_id, run_number)
+
 # Raft-level mosaics of median darks, bias, superflats high and low.
 dark_mosaic = raftTest.RaftMosaic(slot_dependency_glob('*median_dark_bp.fits',
                                                        'bright_defects_raft'),
                                   gains=gains)
-dark_mosaic.plot(title='%s, medianed dark frames' % raft_id)
-plt.savefig('%s_medianed_dark.png' % raft_id)
+dark_mosaic.plot(title='%s, medianed dark frames' % title,
+                 annotation='e-/pixel, gain-corrected, bias-subtracted')
+plt.savefig('%s_medianed_dark.png' % file_prefix)
 del dark_mosaic
 
-mean_bias = raftTest.RaftMosaic(bias_files)
-mean_bias.plot(title='%s, mean bias frames' % raft_id)
-plt.savefig('%s_mean_bias.png' % raft_id)
+mean_bias = raftTest.RaftMosaic(bias_files, bias_subtract=False)
+mean_bias.plot(title='%s, mean bias frames' % title, annotation='ADU/pixel')
+plt.savefig('%s_mean_bias.png' % file_prefix)
 del mean_bias
 
 sflat_high = raftTest.RaftMosaic(slot_dependency_glob('*superflat_high.fits',
                                                       'cte_raft'), gains=gains)
-sflat_high.plot(title='%s, high flux superflat' % raft_id)
-plt.savefig('%s_superflat_high.png' % raft_id)
+sflat_high.plot(title='%s, high flux superflat' % title,
+                annotation='e-/pixel, gain-corrected, bias-subtracted')
+plt.savefig('%s_superflat_high.png' % file_prefix)
 del sflat_high
 
 sflat_low = raftTest.RaftMosaic(slot_dependency_glob('*superflat_low.fits',
                                                      'cte_raft'), gains=gains)
-sflat_low.plot(title='%s, low flux superflat' % raft_id)
-plt.savefig('%s_superflat_low.png' % raft_id)
+sflat_low.plot(title='%s, low flux superflat' % title,
+               annotation='e-/pixel, gain-corrected, bias-subtracted')
+plt.savefig('%s_superflat_low.png' % file_prefix)
 del sflat_low
 
 # QE images at 350, 500, 620, 750, 870, and 1000 nm.
@@ -80,40 +103,56 @@ def slot_dependency_glob(pattern, jobname):
                                  siteUtils.getProcessName('qe_raft_acq'))
     try:
         flat = raftTest.RaftMosaic(files, gains=gains)
-        flat.plot(title='%s, %i nm' % (raft_id, wl))
-        plt.savefig('%s_%04inm_flat.png' % (raft_id, wl))
+        flat.plot(title='%s, %i nm' % (title, wl),
+                  annotation='e-/pixel, gain-corrected, bias-subtracted')
+        plt.savefig('%s_%04inm_flat.png' % (file_prefix, wl))
         del flat
     except IndexError as eobj:
         print(files)
         print(eobj)
 
-# Raft-level bar charts of read noise, nonlinearity, serial and parallel CTI,
+# QE summary plot.
+qe_summary_lims = \
+    siteUtils.dependency_glob('summary.lims', jobname='qe_raft_analysis')[0]
+qe_fig = raftTest.qe_summary_plot(qe_summary_lims, title=title)
+plt.savefig('%s_QE_summary.png' % file_prefix)
+del qe_fig
+
+# Raft-level plots of read noise, nonlinearity, serial and parallel CTI,
 # charge diffusion PSF, and gains from Fe55 and PTC.
-bar_charts = raftTest.RaftBarCharts(results_files)
-
-bar_charts.make_bar_chart('READ_NOISE', 'read noise (-e rms)', spec=9,
-                          title=raft_id)
-plt.savefig('%s_read_noise.png' % raft_id)
-
-bar_charts.make_bar_chart('MAX_FRAC_DEV',
-                          'non-linearity (max. fractional deviation)',
-                          spec=0.02, title=raft_id)
-plt.savefig('%s_linearity.png' % raft_id)
-
-bar_charts.make_multi_bar_chart(('CTI_LOW_SERIAL', 'CTI_HIGH_SERIAL'),
-                                'Serial CTI', spec=5e-6, ylog=True,
-                                title=raft_id, colors='br')
-plt.savefig('%s_serial_cti.png' % raft_id)
-
-bar_charts.make_multi_bar_chart(('CTI_LOW_PARALLEL', 'CTI_HIGH_PARALLEL'),
-                                'Parallel CTI', spec=3e-6, ylog=True,
-                                title=raft_id, colors='br')
-plt.savefig('%s_parallel_cti.png' % raft_id)
-
-bar_charts.make_bar_chart('PSF_SIGMA', 'PSF sigma (microns)', spec=5.,
-                          title=raft_id)
-plt.savefig('%s_psf_sigma.png' % raft_id)
-
-bar_charts.make_multi_bar_chart(('GAIN', 'PTC_GAIN'), 'System Gain (e-/ADU)',
-                                title=raft_id, colors='br', ylog=True)
-plt.savefig('%s_system_gain.png' % raft_id)
+spec_plots = raftTest.RaftSpecPlots(results_files)
+
+columns = 'READ_NOISE DC95_SHOT_NOISE TOTAL_NOISE'.split()
+spec_plots.make_multi_column_plot(columns, 'noise per pixel (-e rms)', spec=9,
+                                  title=title)
+plt.savefig('%s_total_noise.png' % file_prefix)
+
+spec_plots.make_plot('MAX_FRAC_DEV',
+                     'non-linearity (max. fractional deviation)',
+                     spec=0.03, title=title)
+plt.savefig('%s_linearity.png' % file_prefix)
+
+spec_plots.make_multi_column_plot(('CTI_LOW_SERIAL', 'CTI_HIGH_SERIAL'),
+                                  'Serial CTI (ppm)', spec=(5e-6, 3e-5),
+                                  title=title, yscaling=1e6, yerrors=True,
+                                  colors='br', ymax=4e-5)
+plt.savefig('%s_serial_cti.png' % file_prefix)
+
+spec_plots.make_multi_column_plot(('CTI_LOW_PARALLEL', 'CTI_HIGH_PARALLEL'),
+                                  'Parallel CTI (ppm)', spec=3e-6,
+                                  title=title, yscaling=1e6, yerrors=True,
+                                  colors='br')
+plt.savefig('%s_parallel_cti.png' % file_prefix)
+
+spec_plots.make_plot('PSF_SIGMA', 'PSF sigma (microns)', spec=5., title=title,
+                     ymax=5.2)
+plt.savefig('%s_psf_sigma.png' % file_prefix)
+
+spec_plots.make_multi_column_plot(('GAIN', 'PTC_GAIN'), 'System Gain (e-/ADU)',
+                                  yerrors=True, title=title, colors='br')
+plt.savefig('%s_system_gain.png' % file_prefix)
+
+spec_plots.make_plot('DARK_CURRENT_95',
+                     '95th percentile dark current (e-/pixel/s)',
+                     spec=0.2, title=title)
+plt.savefig('%s_dark_current.png' % file_prefix)

harnessed_jobs/cte_raft/v0/producer_cte_raft.py

@@ -10,6 +10,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_cte_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     mask_files = \
         eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
     gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
@@ -49,18 +50,18 @@ def run_cte_task(sensor_id):
         if sflat_file.find('high') != -1:
             flux_level = 'high'
         siteUtils.make_png_file(sensorTest.plot_flat,
-                                sflat_file.replace('.fits', '.png'),
+                                sflat_file.replace('.fits', '.png').replace(sensor_id, file_prefix),
                                 sflat_file,
                                 title=('%s, CTE supeflat, %s flux '
                                        % (sensor_id, flux_level)))
         siteUtils.make_png_file(plots.cte_profiles,
                                 ('%s_serial_oscan_%s.png' %
-                                 (sensor_id, flux_level)),
+                                 (file_prefix, flux_level)),
                                 flux_level, sflat_file, mask_files, serial=True)
 
         siteUtils.make_png_file(plots.cte_profiles,
                                 ('%s_parallel_oscan_%s.png' %
-                                 (sensor_id, flux_level)),
+                                 (file_prefix, flux_level)),
                                 flux_level, sflat_file, mask_files, serial=False)
 
 if __name__ == '__main__':

harnessed_jobs/dark_current_raft/v0/producer_dark_current_raft.py

@@ -9,6 +9,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_dark_current_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     dark_files = siteUtils.dependency_glob('S*/%s_dark_dark_*.fits' % sensor_id,
                                            jobname=siteUtils.getProcessName('dark_raft_acq'),
                                            description='Dark files:')
@@ -30,7 +31,7 @@ def run_dark_current_task(sensor_id):
                                            eo_results['TOTAL_NOISE']))
 
     siteUtils.make_png_file(sensorTest.total_noise_histograms,
-                            '%s_total_noise_hists.png' % sensor_id,
+                            '%s_total_noise_hists.png' % file_prefix,
                             dark_curr_pixels, read_noise, dark95s,
                             exptime=16, title=sensor_id)
 

harnessed_jobs/dark_defects_raft/v0/producer_dark_defects_raft.py

@@ -9,6 +9,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_dark_pixels_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     sflat_files = siteUtils.dependency_glob('S*/%s_sflat_500_flat_H*.fits' % sensor_id,
                                             jobname=siteUtils.getProcessName('sflat_raft_acq'),
                                             description='Superflat files:')
@@ -19,7 +20,7 @@ def run_dark_pixels_task(sensor_id):
     task.run(sensor_id, sflat_files, mask_files)
 
     siteUtils.make_png_file(sensorTest.plot_flat,
-                            '%s_superflat_dark_defects.png' % sensor_id,
+                            '%s_superflat_dark_defects.png' % file_prefix,
                             '%s_median_sflat.fits' % sensor_id,
                             title='%s, superflat for dark defects analysis' % sensor_id)
 

harnessed_jobs/fe55_raft_analysis/v0/producer_fe55_raft_analysis.py

@@ -10,6 +10,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_fe55_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     fe55_files = siteUtils.dependency_glob('S*/%s_fe55_fe55_*.fits' % sensor_id,
                                            jobname=siteUtils.getProcessName('fe55_raft_acq'),
                                            description='Fe55 files:')
@@ -26,7 +27,7 @@ def run_fe55_task(sensor_id):
     #
     print("processing fe55_zoom:", fe55_files[0])
     siteUtils.make_png_file(sensorTest.fe55_zoom,
-                            '%(sensor_id)s_fe55_zoom.png' % locals(),
+                            '%(file_prefix)s_fe55_zoom.png' % locals(),
                             fe55_files[0], size=250, amp=1)
 
     #
@@ -36,18 +37,18 @@ def run_fe55_task(sensor_id):
         pixel_stats = sensorTest.Fe55PixelStats(fe55_files, sensor_id=sensor_id)
 
         siteUtils.make_png_file(pixel_stats.pixel_hists,
-                                '%s_fe55_p3_p5_hists.png' % sensor_id,
+                                '%s_fe55_p3_p5_hists.png' % file_prefix,
                                 pix0='p3', pix1='p5')
 
         siteUtils.make_png_file(pixel_stats.pixel_diff_profile,
-                                '%s_fe55_p3_p5_profiles.png' % sensor_id,
+                                '%s_fe55_p3_p5_profiles.png' % file_prefix,
                                 pixel_coord='x', pix0='p3', pix1='p5')
 
         siteUtils.make_png_file(pixel_stats.apflux_profile,
-                                '%s_fe55_apflux_serial.png' % sensor_id)
+                                '%s_fe55_apflux_serial.png' % file_prefix)
 
         siteUtils.make_png_file(pixel_stats.apflux_profile,
-                                '%s_fe55_apflux_parallel.png' % sensor_id,
+                                '%s_fe55_apflux_parallel.png' % file_prefix,
                                 pixel_coord='y')
 
     except Exception as eobj:
@@ -69,20 +70,20 @@ def run_fe55_task(sensor_id):
     plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
 
     siteUtils.make_png_file(plots.gains,
-                            '%s_gains.png' % sensor_id)
+                            '%s_gains.png' % file_prefix)
 
     siteUtils.make_png_file(sensorTest.plot_flat,
-                            '%s_mean_bias.png' % sensor_id,
+                            '%s_mean_bias.png' % file_prefix,
                             mean_bias_file,
                             title='%s, mean bias frame' % sensor_id)
 
     fe55_file = glob.glob('%s_psf_results*.fits' % sensor_id)[0]
     siteUtils.make_png_file(plots.fe55_dists,
-                            '%s_fe55_dists.png' % sensor_id,
+                            '%s_fe55_dists.png' % file_prefix,
                             fe55_file=fe55_file)
 
     siteUtils.make_png_file(plots.psf_dists,
-                            '%s_psf_dists.png' % sensor_id,
+                            '%s_psf_dists.png' % file_prefix,
                             fe55_file=fe55_file)
 
 if __name__ == '__main__':

harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py

@@ -9,6 +9,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_flat_pair_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     flat_files = siteUtils.dependency_glob('S*/%s_flat*flat?_*.fits' % sensor_id,
                                            jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
                                            description='Flat files:')
@@ -17,20 +18,25 @@ def run_flat_pair_task(sensor_id):
     gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
                                        sensor_id=sensor_id)
 
+    use_exptime = True
     task = sensorTest.FlatPairTask()
     task.run(sensor_id, flat_files, mask_files, gains,
-             linearity_spec_range=(1e4, 9e4))
+             linearity_spec_range=(1e4, 9e4), use_exptime=use_exptime)
 
     results_file = '%s_eotest_results.fits' % sensor_id
     plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
 
+    Ne_bounds = (1e4, 9e4)
+
     detresp_file = '%s_det_response.fits' % sensor_id
     siteUtils.make_png_file(plots.linearity,
-                            '%s_linearity.png' % sensor_id,
-                            detresp_file=detresp_file)
+                            '%s_linearity.png' % file_prefix,
+                            detresp_file=detresp_file, max_dev=0.03,
+                            use_exptime=use_exptime, Ne_bounds=Ne_bounds)
     siteUtils.make_png_file(plots.linearity_resids,
-                            '%s_linearity_resids.png' % sensor_id,
-                            detresp_file=detresp_file)
+                            '%s_linearity_resids.png' % file_prefix,
+                            detresp_file=detresp_file, max_dev=0.03,
+                            Ne_bounds=Ne_bounds, use_exptime=use_exptime)
 
 if __name__ == '__main__':
     sensor_analyses(run_flat_pair_task)

harnessed_jobs/ptc_raft/v0/producer_ptc_raft.py

@@ -9,6 +9,7 @@
 from multiprocessor_execution import sensor_analyses
 
 def run_ptc_task(sensor_id):
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     flat_files = siteUtils.dependency_glob('S*/%s_flat*flat?_*.fits' % sensor_id,
                                            jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
                                            description='Flat files:')
@@ -23,7 +24,7 @@ def run_ptc_task(sensor_id):
     results_file = '%s_eotest_results.fits' % sensor_id
     plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
     siteUtils.make_png_file(plots.ptcs,
-                            '%s_ptcs.png' % sensor_id,
+                            '%s_ptcs.png' % file_prefix,
                             ptc_file='%s_ptc.fits' % sensor_id)
 
 if __name__ == '__main__':

harnessed_jobs/qa_plots_raft/v0/producer_qa_plots_raft.py

@@ -4,6 +4,7 @@
 """
 from __future__ import print_function
 import os
+import glob
 from collections import OrderedDict
 from lcatr.harness.helpers import dependency_glob
 import siteUtils
@@ -44,3 +45,7 @@ def dirname_dependencyGlob(ccd_vendor, **kwds):
     QA_trender.processDirectory(dirname, test_type)
 
 QA_trender.plot(raft_id)
+
+file_prefix = '%s_%s' % (raft_id, siteUtils.getRunNumber())
+for png_file in glob.glob('*QA*.png'):
+    os.rename(png_file, png_file.replace(raft_id, file_prefix))