From 8622183477bd9a1c445c54cac41ec482d740f45c Mon Sep 17 00:00:00 2001
From: Jim Chiang <jchiang@slac.stanford.edu>
Date: Mon, 2 Apr 2018 16:18:27 -0700
Subject: [PATCH 1/7] reimplementation of Aaron's anaNoiseCCD code

---
 python/correlated_noise.py | 120 +++++++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 python/correlated_noise.py

diff --git a/python/correlated_noise.py b/python/correlated_noise.py
new file mode 100644
index 0000000..e4bb3d0
--- /dev/null
+++ b/python/correlated_noise.py
@@ -0,0 +1,120 @@
+import glob
+from collections import namedtuple
+import numpy as np
+import scipy
+import matplotlib.pyplot as plt
+import lsst.afw.image as afw_image
+import lsst.afw.math as afw_math
+import lsst.eotest.image_utils as imutils
+import lsst.eotest.sensor as sensorTest
+
+plt.ion()
+plt.rcParams['xtick.labelsize'] = 'x-small'
+plt.rcParams['ytick.labelsize'] = 'x-small'
+
+
+def get_overscans(infile, oscan_bbox):
+    overscans = dict()
+    for amp in imutils.allAmps():
+        image = afw_image.ImageF(infile, imutils.dm_hdu(amp))
+        overscans[amp] = image.Factory(image, oscan_bbox).getArray()
+    return overscans
+
+
+def get_mean_overscans(infiles, oscan_bbox):
+    mean_overscans = dict()
+    for amp in imutils.allAmps():
+        amp_images = afw_image.vectorImageF()
+        for infile in infiles:
+            amp_images.push_back(afw_image.ImageF(infile, imutils.dm_hdu(amp)))
+        mean_image = afw_math.statisticsStack(amp_images, afw_math.MEAN)
+        mean_overscans[amp] \
+            = mean_image.Factory(mean_image, oscan_bbox).getArray()
+    return mean_overscans
+
+
+BiasStats = namedtuple('BiasStats',
+                       'noise_orig noise_corr corr_factor bias_oscan'.split())
+
+
+def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
+                     figsize=(8, 8), title=''):
+    if make_plots:
+        f1, ax1 = plt.subplots(4, 4, figsize=figsize)
+        ax1 = {amp: subplot for amp, subplot in zip(imutils.allAmps(),
+                                                    ax1.flatten())}
+        f2, ax2 = plt.subplots(4, 4, figsize=figsize)
+        ax2 = {amp: subplot for amp, subplot in zip(imutils.allAmps(),
+                                                    ax2.flatten())}
+
+    # Extract the target filename and omit it from the list of bias files.
+    target_file = bias_files.pop(target)
+
+    # Get the amplifier geometry and serial overscan bounding box from
+    # the FITS headers.
+    amp_geom = sensorTest.makeAmplifierGeometry(target_file)
+    oscan_bbox = amp_geom.serial_overscan
+
+    # Get the target frame overscans.
+    bias_oscans = get_overscans(target_file, oscan_bbox)
+    oscan_shape = bias_oscans[1].shape
+
+    # Construct the mean bias overscans from the remaining files.
+    mean_oscans = get_mean_overscans(bias_files, oscan_bbox)
+
+    # Compute the mean values of the mean bias overscans.
+    mean_oscan_values \
+        = {amp: np.mean(oscan) for amp, oscan in mean_oscans.items()}
+
+    # Loop over amps in target frame and compute statistics.
+    bias_stats = dict()
+    for amp in bias_oscans:
+        print "processing amp", amp
+        # Loop over other amps and construct the mean image of the
+        # bias-subtracted overscans.  Require included amps to have
+        # (unsubtracted) overscans with 4 < stdev < 25 rms ADU.
+        reduced_mean_oscan = np.zeros(oscan_shape)
+        num_oscan = 0
+        for oamp, oscan in bias_oscans.items():
+            if oamp == amp or not (4. < np.std(oscan) < 25):
+                continue
+            reduced_mean_oscan += (oscan - mean_oscans[oamp])
+            num_oscan += 1
+        reduced_mean_oscan -= np.mean(reduced_mean_oscan)
+        reduced_mean_oscan /= num_oscan
+
+        fdata1 = bias_oscans[amp] - mean_oscans[amp]
+        fmean1 = np.mean(fdata1)
+        fdata1 -= fmean1
+        dmat = np.vstack((reduced_mean_oscan.flatten(), fdata1.flatten()))
+        covmat = scipy.cov(dmat, rowvar=True)
+        corr_factor = covmat[0, 1]/covmat[0, 0]
+        fdiff = fdata1 - corr_factor*reduced_mean_oscan
+        bias_stats[amp] = BiasStats(np.sqrt(covmat[1, 1]), np.std(fdiff),
+                                    corr_factor, fmean1)
+
+        if make_plots:
+            f1.suptitle(title)
+            f2.suptitle(title)
+            ax1[amp].hist2d(reduced_mean_oscan.flatten(), fdata1.flatten(),
+                            bins=(100, 100), range=((-50, 50), (-50, 50)))
+            if plot_corr:
+                ax2[amp].hist(fdiff.flatten(), bins=100, range=(-50, 50),
+                              histtype='step')
+            else:
+                ax2[amp].hist(fdata1.flatten(), bins=100, range=(-50, 50),
+                              histtype='step')
+
+    return bias_stats, f1, f2
+
+
+if __name__ == '__main__':
+    bias_files = sorted(glob.glob('/gpfs/slac/lsst/fs1/g/data/jobHarness/jh_archive-test/LCA-11021_RTM/LCA-11021_RTM-002_ETU1/5808D/sflat_raft_acq/v0/38318/S00/ITL-3800C-023-Dev_sflat_bias_*.fits'))
+    etu1_stats, f1, f2 \
+        = correlated_noise(bias_files, make_plots=True, plot_corr=False,
+                           title='Run 5808D, ETU1, S00')
+
+    bias_files = sorted(glob.glob('/gpfs/slac/lsst/fs1/g/data/jobHarness/jh_archive/LCA-11021_RTM/LCA-11021_RTM-005/6288/sflat_raft_acq/v0/37108/S00/*sflat_bias*.fits'))
+    rtm005_stats, f1, f2 \
+        = correlated_noise(bias_files, make_plots=True, plot_corr=False,
+                           title='Run 6288, RTM-005, S00')

From 1229c2409f2c636c5731d38f3a05c88c47b22e42 Mon Sep 17 00:00:00 2001
From: James Chiang <jchiang@slac.stanford.edu>
Date: Tue, 3 Apr 2018 10:17:11 -0700
Subject: [PATCH 2/7] use numpy instead of lsst.afw for computing statistics

---
 python/correlated_noise.py | 69 +++++++++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 24 deletions(-)

diff --git a/python/correlated_noise.py b/python/correlated_noise.py
index e4bb3d0..f05bdf1 100644
--- a/python/correlated_noise.py
+++ b/python/correlated_noise.py
@@ -1,10 +1,9 @@
 import glob
-from collections import namedtuple
+from collections import namedtuple, defaultdict
 import numpy as np
 import scipy
+import astropy.io.fits as fits
 import matplotlib.pyplot as plt
-import lsst.afw.image as afw_image
-import lsst.afw.math as afw_math
 import lsst.eotest.image_utils as imutils
 import lsst.eotest.sensor as sensorTest
 
@@ -13,23 +12,39 @@
 plt.rcParams['ytick.labelsize'] = 'x-small'
 
 
-def get_overscans(infile, oscan_bbox):
+def get_oscan_indices(target_file):
+    "Return the pixel indices of the overscan region."
+    amp_geom = sensorTest.makeAmplifierGeometry(target_file)
+    bbox = amp_geom.serial_overscan
+    return bbox.getMinY(), bbox.getMaxY(), bbox.getMinX(), bbox.getMaxX()
+
+
+def get_overscans(infile, oscan_indices=None):
+    "Return the overscan data as numpy arrays."
+    if oscan_indices is None:
+        y0, y1, x0, x1 = get_oscan_indices(infile)
+    else:
+        y0, y1, x0, x1 = oscan_indices
+    ccd = fits.open(infile)
     overscans = dict()
     for amp in imutils.allAmps():
-        image = afw_image.ImageF(infile, imutils.dm_hdu(amp))
-        overscans[amp] = image.Factory(image, oscan_bbox).getArray()
+        overscans[amp] = ccd[amp].data[y0:y1, x0:x1]
     return overscans
 
 
-def get_mean_overscans(infiles, oscan_bbox):
-    mean_overscans = dict()
-    for amp in imutils.allAmps():
-        amp_images = afw_image.vectorImageF()
-        for infile in infiles:
-            amp_images.push_back(afw_image.ImageF(infile, imutils.dm_hdu(amp)))
-        mean_image = afw_math.statisticsStack(amp_images, afw_math.MEAN)
-        mean_overscans[amp] \
-            = mean_image.Factory(mean_image, oscan_bbox).getArray()
+def get_mean_overscans(infiles, oscan_indices=None):
+    "Compute the mean of the overscans of the list of files"
+    if oscan_indices is None:
+        y0, y1, x0, x1 = get_oscan_indices(infiles[0])
+    else:
+        y0, y1, x0, x1 = oscan_indices
+    mean_overscans = defaultdict(list)
+    for infile in infiles:
+        ccd = fits.open(infile)
+        for amp in imutils.allAmps():
+            mean_overscans[amp].append(ccd[amp].data[y0:y1, x0:x1])
+    for amp, images in mean_overscans.items():
+        mean_overscans[amp] = sum(images)/float(len(images))
     return mean_overscans
 
 
@@ -39,6 +54,11 @@ def get_mean_overscans(infiles, oscan_bbox):
 
 def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
                      figsize=(8, 8), title=''):
+    """
+    Compute the correlated noise statistics for the overscan regions
+    of the list of files, optionally making plots of the distributions.
+    """
+    f1, f2 = None, None
     if make_plots:
         f1, ax1 = plt.subplots(4, 4, figsize=figsize)
         ax1 = {amp: subplot for amp, subplot in zip(imutils.allAmps(),
@@ -50,17 +70,13 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
     # Extract the target filename and omit it from the list of bias files.
     target_file = bias_files.pop(target)
 
-    # Get the amplifier geometry and serial overscan bounding box from
-    # the FITS headers.
-    amp_geom = sensorTest.makeAmplifierGeometry(target_file)
-    oscan_bbox = amp_geom.serial_overscan
 
     # Get the target frame overscans.
-    bias_oscans = get_overscans(target_file, oscan_bbox)
+    bias_oscans = get_overscans(target_file)
     oscan_shape = bias_oscans[1].shape
 
     # Construct the mean bias overscans from the remaining files.
-    mean_oscans = get_mean_overscans(bias_files, oscan_bbox)
+    mean_oscans = get_mean_overscans(bias_files)
 
     # Compute the mean values of the mean bias overscans.
     mean_oscan_values \
@@ -69,7 +85,6 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
     # Loop over amps in target frame and compute statistics.
     bias_stats = dict()
     for amp in bias_oscans:
-        print "processing amp", amp
         # Loop over other amps and construct the mean image of the
         # bias-subtracted overscans.  Require included amps to have
         # (unsubtracted) overscans with 4 < stdev < 25 rms ADU.
@@ -91,7 +106,9 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
         corr_factor = covmat[0, 1]/covmat[0, 0]
         fdiff = fdata1 - corr_factor*reduced_mean_oscan
         bias_stats[amp] = BiasStats(np.sqrt(covmat[1, 1]), np.std(fdiff),
-                                    corr_factor, fmean1)
+                                    corr_factor,
+                                    np.mean(bias_oscans[amp]))
+                                    #fmean1)
 
         if make_plots:
             f1.suptitle(title)
@@ -111,10 +128,14 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
 if __name__ == '__main__':
     bias_files = sorted(glob.glob('/gpfs/slac/lsst/fs1/g/data/jobHarness/jh_archive-test/LCA-11021_RTM/LCA-11021_RTM-002_ETU1/5808D/sflat_raft_acq/v0/38318/S00/ITL-3800C-023-Dev_sflat_bias_*.fits'))
     etu1_stats, f1, f2 \
-        = correlated_noise(bias_files, make_plots=True, plot_corr=False,
+        = correlated_noise(bias_files, make_plots=False, plot_corr=False,
                            title='Run 5808D, ETU1, S00')
+    for amp, stats in etu1_stats.items():
+        print amp, stats
 
     bias_files = sorted(glob.glob('/gpfs/slac/lsst/fs1/g/data/jobHarness/jh_archive/LCA-11021_RTM/LCA-11021_RTM-005/6288/sflat_raft_acq/v0/37108/S00/*sflat_bias*.fits'))
     rtm005_stats, f1, f2 \
         = correlated_noise(bias_files, make_plots=True, plot_corr=False,
                            title='Run 6288, RTM-005, S00')
+    for amp, stats in rtm005_stats.items():
+        print amp, stats

From 4d1808fe8d8320b32017ec56692bd52aa3617cdc Mon Sep 17 00:00:00 2001
From: Jim Chiang <jchiang@slac.stanford.edu>
Date: Mon, 14 May 2018 10:11:14 -0700
Subject: [PATCH 3/7] add docstring; add correlated_noise analysis to producer
 script

---
 .../v0/producer_read_noise_raft.py            |  9 +++++++
 python/correlated_noise.py                    | 25 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
index 83d385b..71676ba 100755
--- a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
@@ -8,6 +8,7 @@
 import eotestUtils
 from multiprocessor_execution import sensor_analyses
 
+
 def run_read_noise_task(sensor_id):
     file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
     bias_files = siteUtils.dependency_glob('S*/%s_fe55_fe55_*.fits' % sensor_id,
@@ -26,5 +27,13 @@ def run_read_noise_task(sensor_id):
     task.run(sensor_id, bias_files, gains, system_noise=system_noise,
              mask_files=mask_files, use_overscan=True)
 
+    # Compute noise correlation statistics.
+    bias_stats, corr_fig, hist_fig \
+        = correlated_noise(bias_files, target=0, make_plots=True,
+                           title=sensor_id)
+    plt.figure(corr_fig.number)
+    plt.savefig('%s_correlated_noise.png' % file_prefix)
+
+
 if __name__ == '__main__':
     sensor_analyses(run_read_noise_task)
diff --git a/python/correlated_noise.py b/python/correlated_noise.py
index f05bdf1..c36a4af 100644
--- a/python/correlated_noise.py
+++ b/python/correlated_noise.py
@@ -57,6 +57,30 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
     """
     Compute the correlated noise statistics for the overscan regions
     of the list of files, optionally making plots of the distributions.
+
+    Parameters
+    ----------
+    bias_files: list
+        List of bias files to analyze.  This list must have at least as many
+        files as the target file index + 1.
+    target: int
+        Bias frame to compare to the mean biases constructed from the
+        remaining files.
+    make_plots: bool [False]
+        Flag to determine if the png plots will be generated.
+    plot_corr: bool [True]
+        Flag to plot the histograms of correlated pixel values.  If False,
+        then plot histograms of the pixel values of the mean-subtracted
+        target frame.
+    figsize: tuple [(8, 8)]
+        Figure size (in inches) of 4x4 grid of correlation plots.
+    title: str ['']
+        Title of 4x4 grid.
+
+    Returns
+    -------
+    (BiasStats, figure, figure):  tuple of bias stats results and matplotlib
+        figures.
     """
     f1, f2 = None, None
     if make_plots:
@@ -70,7 +94,6 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
     # Extract the target filename and omit it from the list of bias files.
     target_file = bias_files.pop(target)
 
-
     # Get the target frame overscans.
     bias_oscans = get_overscans(target_file)
     oscan_shape = bias_oscans[1].shape

From 1a96b02919c4733bf3d5a91883f3809437130abe Mon Sep 17 00:00:00 2001
From: James Chiang <jchiang@slac.stanford.edu>
Date: Mon, 14 May 2018 13:04:04 -0700
Subject: [PATCH 4/7] persist correlated_noise results in validator script

---
 .../v0/producer_read_noise_raft.py                | 11 +++++++----
 .../v0/validator_read_noise_raft.py               | 13 +++++++++++++
 python/correlated_noise.py                        | 15 ++++++++++-----
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
index 71676ba..62fb417 100755
--- a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
@@ -3,9 +3,11 @@
 Producer script for raft-level read noise analysis.
 """
 from __future__ import print_function
+import pickle
 import lsst.eotest.sensor as sensorTest
 import siteUtils
 import eotestUtils
+from correlated_noise import correlated_noise
 from multiprocessor_execution import sensor_analyses
 
 
@@ -27,13 +29,14 @@ def run_read_noise_task(sensor_id):
     task.run(sensor_id, bias_files, gains, system_noise=system_noise,
              mask_files=mask_files, use_overscan=True)
 
-    # Compute noise correlation statistics.
-    bias_stats, corr_fig, hist_fig \
-        = correlated_noise(bias_files, target=0, make_plots=True,
-                           title=sensor_id)
+    # Compute amp-amp correlated noise.
+    bias_stats, corr_fig, _ = correlated_noise(bias_files, target=0,
+                                               make_plots=True, title=sensor_id)
     plt.figure(corr_fig.number)
     plt.savefig('%s_correlated_noise.png' % file_prefix)
 
+    with open('%s_noise_stats.pkl' % sensor_id, 'wb') as output:
+        pickle.dump(bias_stats, output)
 
 if __name__ == '__main__':
     sensor_analyses(run_read_noise_task)
diff --git a/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
index 7ff4e4f..b916df4 100755
--- a/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
@@ -4,6 +4,7 @@
 """
 from __future__ import print_function
 import glob
+import pickle
 import lsst.eotest.sensor as sensorTest
 import lcatr.schema
 import siteUtils
@@ -45,6 +46,18 @@
                      for item in files]
     results.extend(data_products)
 
+    # Add the correlated_noise results.
+    cn_schema = lcatr.schema.get('correlated_noise')
+    with open('%s_noise_stats.pkl' % sensor_id, 'rb') as input_:
+        bias_stats = pickle.load(input_)
+        for amp in bias_stats:
+            results.append(
+                lcatr.schema.valid(cn_schema, amp=amp,
+                                   slot=slot, sensor_id=sensor_id,
+                                   noise_rms_orig=bias_stats.noise_orig,
+                                   noise_rms_corrected=bias_stats.nosie_corr,
+                                   correction_factor=bias_stats.corr_factor))
+
     # Persist the png files.
     metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id,
                     TESTTYPE='FE55', TEST_CATEGORY='EO')
diff --git a/python/correlated_noise.py b/python/correlated_noise.py
index c36a4af..5a1af05 100644
--- a/python/correlated_noise.py
+++ b/python/correlated_noise.py
@@ -69,9 +69,9 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
     make_plots: bool [False]
         Flag to determine if the png plots will be generated.
     plot_corr: bool [True]
-        Flag to plot the histograms of correlated pixel values.  If False,
-        then plot histograms of the pixel values of the mean-subtracted
-        target frame.
+        Flag to plot the histograms of correlation-corrected pixel
+        values.  If False, then plot histograms of the uncorrected pixel
+        values.
     figsize: tuple [(8, 8)]
         Figure size (in inches) of 4x4 grid of correlation plots.
     title: str ['']
@@ -79,8 +79,13 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
 
     Returns
     -------
-    (BiasStats, figure, figure):  tuple of bias stats results and matplotlib
-        figures.
+    (dict, figure, figure):  tuple of results and matplotlib
+        figures.  The first item is a dict of BiasStats objects,
+
+        BiasStats = namedtuple('BiasStats', \
+                    'noise_orig noise_corr corr_factor bias_oscan'.split())
+
+        that contain the results for each amplifier.
     """
     f1, f2 = None, None
     if make_plots:

From 689945111444fdce47b364ffded925829e3da703 Mon Sep 17 00:00:00 2001
From: James Chiang <jchiang@slac.stanford.edu>
Date: Mon, 14 May 2018 13:18:54 -0700
Subject: [PATCH 5/7] add correlated_noise.schema; persist only the
 covariance-variance ratio

---
 .../read_noise_raft/v0/validator_read_noise_raft.py      | 4 +---
 schemas/correlated_noise.schema                          | 9 +++++++++
 2 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100644 schemas/correlated_noise.schema

diff --git a/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
index b916df4..1fdb064 100755
--- a/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
@@ -54,9 +54,7 @@
             results.append(
                 lcatr.schema.valid(cn_schema, amp=amp,
                                    slot=slot, sensor_id=sensor_id,
-                                   noise_rms_orig=bias_stats.noise_orig,
-                                   noise_rms_corrected=bias_stats.nosie_corr,
-                                   correction_factor=bias_stats.corr_factor))
+                                   cov_var_ratio=bias_stats.corr_factor))
 
     # Persist the png files.
     metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id,
diff --git a/schemas/correlated_noise.schema b/schemas/correlated_noise.schema
new file mode 100644
index 0000000..ee49f6c
--- /dev/null
+++ b/schemas/correlated_noise.schema
@@ -0,0 +1,9 @@
+# -*- python -*-
+{
+    'schema_name' : 'correlated_noise',
+    'schema_version' : 0,
+    'amp' : int,
+    'slot' : str,
+    'sensor_id' : str,
+    'cov_var_ratio' : float
+}

From 19f72cdf72f31ac93570042c49b5924510c05c01 Mon Sep 17 00:00:00 2001
From: Jim Chiang <jchiang@slac.stanford.edu>
Date: Mon, 14 May 2018 14:41:40 -0700
Subject: [PATCH 6/7] remove code to write correlated_noise results to eT
 tables

---
 .../read_noise_raft/v0/producer_read_noise_raft.py   |  4 ----
 .../read_noise_raft/v0/validator_read_noise_raft.py  | 11 -----------
 python/correlated_noise.py                           | 12 ++++++++++--
 schemas/correlated_noise.schema                      |  9 ---------
 4 files changed, 10 insertions(+), 26 deletions(-)
 delete mode 100644 schemas/correlated_noise.schema

diff --git a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
index 62fb417..c70b470 100755
--- a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
@@ -3,7 +3,6 @@
 Producer script for raft-level read noise analysis.
 """
 from __future__ import print_function
-import pickle
 import lsst.eotest.sensor as sensorTest
 import siteUtils
 import eotestUtils
@@ -35,8 +34,5 @@ def run_read_noise_task(sensor_id):
     plt.figure(corr_fig.number)
     plt.savefig('%s_correlated_noise.png' % file_prefix)
 
-    with open('%s_noise_stats.pkl' % sensor_id, 'wb') as output:
-        pickle.dump(bias_stats, output)
-
 if __name__ == '__main__':
     sensor_analyses(run_read_noise_task)
diff --git a/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
index 1fdb064..7ff4e4f 100755
--- a/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/validator_read_noise_raft.py
@@ -4,7 +4,6 @@
 """
 from __future__ import print_function
 import glob
-import pickle
 import lsst.eotest.sensor as sensorTest
 import lcatr.schema
 import siteUtils
@@ -46,16 +45,6 @@
                      for item in files]
     results.extend(data_products)
 
-    # Add the correlated_noise results.
-    cn_schema = lcatr.schema.get('correlated_noise')
-    with open('%s_noise_stats.pkl' % sensor_id, 'rb') as input_:
-        bias_stats = pickle.load(input_)
-        for amp in bias_stats:
-            results.append(
-                lcatr.schema.valid(cn_schema, amp=amp,
-                                   slot=slot, sensor_id=sensor_id,
-                                   cov_var_ratio=bias_stats.corr_factor))
-
     # Persist the png files.
     metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id,
                     TESTTYPE='FE55', TEST_CATEGORY='EO')
diff --git a/python/correlated_noise.py b/python/correlated_noise.py
index 5a1af05..1f81a1c 100644
--- a/python/correlated_noise.py
+++ b/python/correlated_noise.py
@@ -7,7 +7,6 @@
 import lsst.eotest.image_utils as imutils
 import lsst.eotest.sensor as sensorTest
 
-plt.ion()
 plt.rcParams['xtick.labelsize'] = 'x-small'
 plt.rcParams['ytick.labelsize'] = 'x-small'
 
@@ -143,6 +142,10 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
             f2.suptitle(title)
             ax1[amp].hist2d(reduced_mean_oscan.flatten(), fdata1.flatten(),
                             bins=(100, 100), range=((-50, 50), (-50, 50)))
+            label = 'amp %i, cov/var = %.2f' \
+                    % (amp, bias_stats[amp].corr_factor)
+            ax1[amp].text(-40, 40, label, fontsize=6, color='w',
+                          fontweight='bold')
             if plot_corr:
                 ax2[amp].hist(fdiff.flatten(), bins=100, range=(-50, 50),
                               histtype='step')
@@ -154,12 +157,15 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
 
 
 if __name__ == '__main__':
+    plt.ion()
     bias_files = sorted(glob.glob('/gpfs/slac/lsst/fs1/g/data/jobHarness/jh_archive-test/LCA-11021_RTM/LCA-11021_RTM-002_ETU1/5808D/sflat_raft_acq/v0/38318/S00/ITL-3800C-023-Dev_sflat_bias_*.fits'))
     etu1_stats, f1, f2 \
-        = correlated_noise(bias_files, make_plots=False, plot_corr=False,
+        = correlated_noise(bias_files, make_plots=True, plot_corr=False,
                            title='Run 5808D, ETU1, S00')
     for amp, stats in etu1_stats.items():
         print amp, stats
+    plt.figure(f1.number)
+    plt.savefig('ETU1_S00_noise_corr.png')
 
     bias_files = sorted(glob.glob('/gpfs/slac/lsst/fs1/g/data/jobHarness/jh_archive/LCA-11021_RTM/LCA-11021_RTM-005/6288/sflat_raft_acq/v0/37108/S00/*sflat_bias*.fits'))
     rtm005_stats, f1, f2 \
@@ -167,3 +173,5 @@ def correlated_noise(bias_files, target=0, make_plots=False, plot_corr=True,
                            title='Run 6288, RTM-005, S00')
     for amp, stats in rtm005_stats.items():
         print amp, stats
+    plt.figure(f1.number)
+    plt.savefig('RTM-005_S00_noise_corr.png')
diff --git a/schemas/correlated_noise.schema b/schemas/correlated_noise.schema
deleted file mode 100644
index ee49f6c..0000000
--- a/schemas/correlated_noise.schema
+++ /dev/null
@@ -1,9 +0,0 @@
-# -*- python -*-
-{
-    'schema_name' : 'correlated_noise',
-    'schema_version' : 0,
-    'amp' : int,
-    'slot' : str,
-    'sensor_id' : str,
-    'cov_var_ratio' : float
-}

From aafa6635c41a1dd88726b446af47e6f1903f8ddc Mon Sep 17 00:00:00 2001
From: Jim Chiang <jchiang@slac.stanford.edu>
Date: Mon, 14 May 2018 17:06:21 -0700
Subject: [PATCH 7/7] import matplotlib.pyplot

---
 harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
index c70b470..c091491 100755
--- a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
+++ b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py
@@ -3,6 +3,7 @@
 Producer script for raft-level read noise analysis.
 """
 from __future__ import print_function
+import matplotlib.pyplot as plt
 import lsst.eotest.sensor as sensorTest
 import siteUtils
 import eotestUtils