Merge pull request #50 from lsst-camera-dh/LSSTTD-1444_missing_CCDs_i…

…n_raft_noise_correlation

handle missing CCDs in raft noise correlation calculation

harnessed_jobs/cti_BOT/v0/cte_jh_task.py

@@ -29,9 +29,13 @@ def cte_jh_task(det_name):
 
     eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
     gains = get_amplifier_gains(eotest_results_file)
-    results_file = siteUtils.dependency_glob(eotest_results_file,
-                                             jobname='fe55_analysis_BOT')[0]
-    shutil.copy(results_file, eotest_results_file)
+    try:
+        results_file = siteUtils.dependency_glob(eotest_results_file,
+                                                 jobname='fe55_analysis_BOT')[0]
+    except IndexError:
+        pass
+    else:
+        shutil.copy(results_file, eotest_results_file)
 
     bias_frame = bias_filename(file_prefix)
 

harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py

@@ -22,14 +22,7 @@ def flat_pairs_jh_task(det_name):
         return None
 
     mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix)))
-    try:
-        eotest_results_file \
-            = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix),
-                                        jobname='fe55_analysis_BOT')[0]
-    except IndexError:
-        print("flat_pairs_jh_task: Fe55 eotest results file not found for ",
-              file_prefix, ".  Using unit gains.")
-        eotest_results_file = None
+    eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
     gains = get_amplifier_gains(eotest_results_file)
     bias_frame = bias_filename(file_prefix)
 

harnessed_jobs/raft_results_summary_BOT/v0/producer_raft_results_summary_BOT.py

@@ -79,7 +79,8 @@ def make_focal_plane_plots():
         = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs',
                        'raft_results_summary_BOT', 'v0', 'raft_results_task.py')
 
-    installed_rafts = camera_info.get_installed_raft_names()
+#    installed_rafts = camera_info.get_installed_raft_names()
+    installed_rafts = camera_info.installed_science_rafts
     run_python_task_or_cl_script(raft_results_task, raft_results_task_script,
                                  device_names=installed_rafts)
 

harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py

@@ -32,11 +32,16 @@ def plt_savefig(filename):
     # compliance.  Use one of the median bias files for this since they
     # should be available no matter which analysis tasks are run.
     bias_frames = get_raft_files_by_slot(raft_name, 'median_bias.fits')
-    mask_files = get_raft_files_by_slot(raft_name, 'edge_rolloff_mask.fits')
-
+    try:
+        mask_files = get_raft_files_by_slot(raft_name,
+                                            'edge_rolloff_mask.fits')
+    except FileNotFoundError:
+        input_mask = None
+    else:
+        input_mask = list(mask_files.values())[0]
     total_num, rolloff_mask \
         = sensorTest.pixel_counts(list(bias_frames.values())[0],
-                                  input_mask=list(mask_files.values())[0])
+                                  input_mask=input_mask)
 
     # Exposure time (in seconds) for 95th percentile dark current shot
     # noise calculation.
@@ -55,7 +60,11 @@ def plt_savefig(filename):
                 eotest_results.add_seg_result(amp, 'MAX_FRAC_DEV', 0.)
             eotest_results.add_seg_result(amp, 'DC95_SHOT_NOISE',
                                           np.float(shot_noise[i]))
-            eotest_results['TOTAL_NOISE'][i] = total_noise[i]
+            try:
+                eotest_results['TOTAL_NOISE'][i] = total_noise[i]
+            except KeyError:
+                eotest_results.add_seg_result(amp, 'TOTAL_NOISE',
+                                              np.float(total_noise[i]))
         eotest_results.write(filename)
 
     run = siteUtils.getRunNumber()

harnessed_jobs/read_noise_BOT/v0/raft_jh_noise_correlations.py

@@ -33,10 +33,14 @@ def raft_jh_noise_correlations(raft_name):
         # slot_name is the last '_'-delimited field before '.fits'.
         slot_name = item.split('_')[-1].split('.')[0]
         bias_frame_dict[seqno][slot_name] = item
+    bias_frame_files = dict()
     for seqno in bias_frame_dict:
         if len(bias_frame_dict[seqno]) == 9:
             bias_frame_files = bias_frame_dict[seqno]
             break
+        else:
+            if len(bias_frame_dict[seqno]) > len(bias_frame_files):
+                bias_frame_files = bias_frame_dict[seqno]
     logger.info("bias frame files for raft_noise_correlations:")
     for key, value in bias_frame_files.items():
         logger.info("   %s", value)

python/bot_eo_analyses.py

@@ -209,7 +209,7 @@ def __call__(self, file_pattern):
         if self.run is None:
             return _get_amplifier_gains(file_pattern)
         # Extract the det_name from the file pattern.
-        match = re.search('R\d\d_S\d\d', file_pattern)
+        match = re.search('R\d\d_S\w\d', file_pattern)
         if match is None:
             message = f"no det_name match in {file_pattern}"
             raise RuntimeError("GetAmplifierGains.__call__: " + message)

python/correlated_noise.py

@@ -195,16 +195,20 @@ def raft_level_oscan_correlations(bias_files, buffer=10, title='',
         the numpy array containing the correlation coefficients.
     """
     slots = 'S00 S01 S02 S10 S11 S12 S20 S21 S22'.split()
-    bbox = None
     overscans = []
+
+    ccd0 = sensorTest.MaskedCCD(list(bias_files.values())[0])
+    bbox = ccd0.amp_geom.serial_overscan
+    bbox.grow(-buffer)
     for slot in slots:
-        ccd = sensorTest.MaskedCCD(bias_files[slot])
-        if bbox is None:
-            bbox = ccd.amp_geom.serial_overscan
-            bbox.grow(-buffer)
-        for amp in ccd:
-            image = ccd[amp].getImage()
-            overscans.append(image.Factory(image, bbox).getArray())
+        if slot not in bias_files:
+            for amp in ccd0:
+                overscans.append(np.zeros((bbox.getHeight(), bbox.getWidth())))
+        else:
+            ccd = sensorTest.MaskedCCD(bias_files[slot])
+            for amp in ccd:
+                image = ccd[amp].getImage()
+                overscans.append(image.Factory(image, bbox).getArray())
     namps = len(overscans)
     data = np.array([np.corrcoef(overscans[i[0]].ravel(),
                                  overscans[i[1]].ravel())[0, 1]

python/multiprocessor_execution.py

@@ -12,7 +12,7 @@
     from parsl_execution import parsl_sensor_analyses, \
         parsl_device_analysis_pool
 except ImportError as eobj:
-    warnings.warn(f'ImportError: {eobj}')
+    #warnings.warn(f'ImportError: {eobj}')
     pass
 from ssh_dispatcher import ssh_device_analysis_pool
 
@@ -79,7 +79,9 @@ def run_device_analysis_pool(task_func, device_names, processes=None, cwd=None,
                                           processes=processes, cwd=cwd)
 
     if siteUtils.getUnitType() == 'LCA-10134_Cryostat':
-        return ssh_device_analysis_pool(task_func, device_names, cwd=cwd)
+        max_time = os.environ.get('LCATR_MAX_JOB_TIME', None)
+        return ssh_device_analysis_pool(task_func, device_names, cwd=cwd,
+                                        max_time=max_time)
 
     if processes is None:
         # Use the maximum number of cores available, reserving one for