Merge pull request #14 from lsst-camera-dh/LSSTTD-1273

tested in fake eTraveler environment

harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py

@@ -14,6 +14,7 @@
 import eotestUtils
 import siteUtils
 import camera_components
+from tearing_detection import tearing_detection
 
 def slot_dependency_glob(pattern, jobname):
     "Return an OrderedDict of files with the desired pattern, keyed by slot."
@@ -30,10 +31,11 @@ def slot_dependency_glob(pattern, jobname):
 
 # Exposure time (in seconds) for 95th percentile dark current shot
 # noise calculation.
-exptime = 16.
+exptime = 15.
 
 raft_id = siteUtils.getUnitId()
 raft = camera_components.Raft.create_from_etrav(raft_id)
+sensor_ids = {slot: sensor_id for slot, sensor_id in raft.items()}
 summary_files = dependency_glob('summary.lims')
 results_files = dict()
 for slot, sensor_id in raft.items():
@@ -49,7 +51,7 @@ def slot_dependency_glob(pattern, jobname):
     # to read noise to produce updated total noise.
     shot_noise = repackager.eotest_results['DARK_CURRENT_95']*exptime
     total_noise = np.sqrt(repackager.eotest_results['READ_NOISE']**2
-                          + shot_noise**2)
+                          + shot_noise)
     for i, amp in enumerate(repackager.eotest_results['AMP']):
         repackager.eotest_results.add_seg_result(amp, 'DC95_SHOT_NOISE',
                                                  np.float(shot_noise[i]))

harnessed_jobs/tearing_detection/v0/modulefile

@@ -0,0 +1,3 @@
+#%Module1.0 #-*-tcl-*-#
+source "$::env(LCATR_MODULES)/lcatr.tcl"
+lcatr_package producer_tearing_detection.py validator_tearing_detection.py

harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Producer script for raft-level flat pairs analysis.
+"""
+from __future__ import print_function
+import pickle
+import siteUtils
+from multiprocessor_execution import sensor_analyses
+from tearing_detection import tearing_detection
+
+acq_jobs = {('flat_pair_raft_acq', 'N/A'): 'S*/%s_flat*flat?_*.fits',
+            ('qe_raft_acq', 'N/A'): 'S*/%s_lambda_flat_*.fits',
+            ('sflat_raft_acq', 'low_flux'): 'S*/%s_sflat_500_flat_L*.fits',
+            ('sflat_raft_acq', 'high_flux'): 'S*/%s_sflat_500_flat_H*.fits'}
+
+def run_tearing_detection(sensor_id):
+    """
+    Loop over the acquisition jobs and perform tearing analysis on each.
+    """
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
+    tearing_stats = []
+    for job_key, pattern in acq_jobs.items():
+        job_name, subset = job_key
+        flats = siteUtils.dependency_glob(pattern % sensor_id,
+                                          jobname=siteUtils.getProcessName(job_name),
+                                          description='Flat files:')
+        tearing_found, _ = tearing_detection(flats)
+        tearing_stats.append((job_name, subset, sensor_id, len(tearing_found)))
+
+    with open('%s_tearing_stats.pkl' % file_prefix, 'wb') as output:
+        pickle.dump(tearing_stats, output)
+
+if __name__ == '__main__':
+    sensor_analyses(run_tearing_detection)

harnessed_jobs/tearing_detection/v0/validator_tearing_detection.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""
+Validator script for raft-level flat pairs analysis.
+"""
+import pickle
+import lcatr.schema
+import siteUtils
+import camera_components
+
+raft_id = siteUtils.getUnitId()
+raft = camera_components.Raft.create_from_etrav(raft_id)
+
+results = []
+schema = lcatr.schema.get('tearing_detection')
+for slot, sensor_id in raft.items():
+    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
+    with open('%s_tearing_stats.pkl' % file_prefix, 'rb') as input_:
+        tearing_stats = pickle.load(input_)
+    for values in tearing_stats:
+        stats = dict(_ for _ in zip(('job_name', 'subset', 'sensor_id',
+                                     'detections', 'slot'),
+                                    list(values) + [slot]))
+        results.append(lcatr.schema.valid(schema, **stats))
+
+results.extend(siteUtils.jobInfo())
+lcatr.schema.write_file(results)
+lcatr.schema.validate_file()

python/tearing_detection.py

@@ -0,0 +1,75 @@
+"""
+Functions to process raft-level single sensor flats for tearing detection.
+"""
+import os
+import matplotlib.pyplot as plt
+import lsst.eotest.sensor as sensorTest
+import siteUtils
+
+__all__ = ['tearing_detection', 'persist_tearing_png_files']
+
+def tearing_detection(fitsfiles, make_png_files=False):
+    """
+    Run the tearing detection code over a collection of single
+    sensor flats.
+
+    Parameters
+    ----------
+    fitsfiles: list
+        List of single sensor flats.  This should generally be for a
+        given sensor since any png files of the tearing profiles would
+        be later persisted by slot number.
+    make_png_files: bool [False]
+        Flag to make png files of the tearing profiles.
+
+    Returns
+    -------
+    tuple(list, list): The first item is a list of files for which
+    tearing has been detected.  The second item is a list of png files
+    of the tearing profiles for the sensors with tearing, if requested.
+    """
+    files_with_tearing = []
+    png_files = []
+    for filename in fitsfiles:
+        ts = sensorTest.TearingStats(filename)
+        if ts.has_tearing():
+            files_with_tearing.append(filename)
+            if make_png_files:
+                ts.plot_profiles()
+                outfile = (os.path.basename(filename).split('.')[0] +
+                           '_tearing.png')
+                plt.savefig(outfile)
+                png_files.append(outfile)
+    return files_with_tearing, png_files
+
+
+def persist_tearing_png_files(png_files, folder=None, metadata=None):
+    """
+    Create the lcatr.schema.filerefs for persisting the png files
+    with the DataCatalog.
+
+    Parameters
+    ----------
+    png_files: list
+        A list of png files for the tearing profile plots.
+    folder: str [None]
+        Folder under which to persist the png file.  For raft-level
+        analysis, this would be the slot number of the CCD.
+    metadata: dict [None]
+        Any additional metadata to persist with the png files.
+
+    Returns
+    -------
+    list: This is a list of lcatr.schema.filerefs.
+    """
+    if metadata is None:
+        metadata = dict()
+    md = siteUtils.DataCatalogMetadata(**metadata)
+    png_filerefs = []
+    for png_file in png_files:
+        dp = 'tearing_profiles'
+        lsst_id = os.path.basename(png_file).split('_')[0]
+        png_filerefs.append(siteUtils.make_fileref(png_file, folder=folder,
+                                                   metadata=md(DATA_PRODUCT=dp,
+                                                               LsstId=lsst_id)))
+    return png_filerefs

schemas/collect_raft_results.schema

@@ -0,0 +1,10 @@
+# -*- python -*-
+{
+    'schema_name' : 'tearing_detection',
+    'schema_version' : 0,
+    'job_name' : str,
+    'subset' : str,
+    'slot' : str,
+    'sensor_id' : str,
+    'detections': int
+}

tests/test_tearing_detection.py

@@ -0,0 +1,53 @@
+"""
+Unit tests for tearing detection code.
+"""
+import os
+import unittest
+from tearing_detection import tearing_detection, persist_tearing_png_files
+
+class TearingDetectionTestCase(unittest.TestCase):
+    """
+    TestCase class for tearing_detection module.
+    """
+    def setUp(self):
+        self.flat_with_tearing = 'E2V-CCD250-229-Dev_flat_with_tearing.fits.gz'
+        self.superflat_without_tearing \
+            = 'E2V-CCD250-160_sflat_without_tearing.fits.gz'
+        self.png_filename \
+            = self.flat_with_tearing.split('.')[0] + '_tearing.png'
+
+    def tearDown(self):
+        if os.path.isfile(self.png_filename):
+            os.remove(self.png_filename)
+
+    def test_tearing_detection(self):
+        """
+        Test for tearing_detection and persist_tearing_png_files functions.
+        """
+        fits_files = [os.path.join(os.environ['EOANALYSISJOBSDIR'], 'tests',
+                                   'data', x) for x in
+                      (self.flat_with_tearing, self.superflat_without_tearing)]
+        files_with_tearing, png_files = tearing_detection(fits_files)
+        # Check that png_files is empty.
+        self.assertFalse(png_files)
+        # Check that the expected filename is returned.
+        self.assertEqual(files_with_tearing[0], fits_files[0])
+        self.assertEqual(len(files_with_tearing), 1)
+
+        # Check output if make_png_files is True.
+        files_with_tearing, png_files \
+            = tearing_detection(fits_files, make_png_files=True)
+        # Check that the expected filenames are returned.
+        self.assertListEqual(files_with_tearing, [fits_files[0]])
+        self.assertListEqual(png_files, [self.png_filename])
+
+        # Test the persistence function for the png files.
+        file_refs = persist_tearing_png_files(png_files)
+        self.assertEqual(len(file_refs), 1)
+        self.assertEqual(file_refs[0]['path'], png_files[0])
+        md = eval(file_refs[0]['metadata'])
+        self.assertEqual(md['DATA_PRODUCT'], 'tearing_profiles')
+        self.assertEqual(md['LsstId'], 'E2V-CCD250-229-Dev')
+
+if __name__ == '__main__':
+    unittest.main()