From 193335f17bad4ff10fd236f6f6280aa40815b51f Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 5 Oct 2019 16:31:19 -0700 Subject: [PATCH 1/6] handle missing CCDs in raft noise correlation calculation --- .../v0/raft_jh_noise_correlations.py | 4 ++++ python/correlated_noise.py | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/harnessed_jobs/read_noise_BOT/v0/raft_jh_noise_correlations.py b/harnessed_jobs/read_noise_BOT/v0/raft_jh_noise_correlations.py index f9bc3d3..8621171 100755 --- a/harnessed_jobs/read_noise_BOT/v0/raft_jh_noise_correlations.py +++ b/harnessed_jobs/read_noise_BOT/v0/raft_jh_noise_correlations.py @@ -33,10 +33,14 @@ def raft_jh_noise_correlations(raft_name): # slot_name is the last '_'-delimited field before '.fits'. slot_name = item.split('_')[-1].split('.')[0] bias_frame_dict[seqno][slot_name] = item + bias_frame_files = dict() for seqno in bias_frame_dict: if len(bias_frame_dict[seqno]) == 9: bias_frame_files = bias_frame_dict[seqno] break + else: + if len(bias_frame_dict[seqno]) > len(bias_frame_files): + bias_frame_files = bias_frame_dict[seqno] logger.info("bias frame files for raft_noise_correlations:") for key, value in bias_frame_files.items(): logger.info(" %s", value) diff --git a/python/correlated_noise.py b/python/correlated_noise.py index 0c62fb2..2a6bd90 100644 --- a/python/correlated_noise.py +++ b/python/correlated_noise.py @@ -195,16 +195,20 @@ def raft_level_oscan_correlations(bias_files, buffer=10, title='', the numpy array containing the correlation coefficients. """ slots = 'S00 S01 S02 S10 S11 S12 S20 S21 S22'.split() - bbox = None overscans = [] + + ccd0 = sensorTest.MaskedCCD(list(bias_files.values())[0]) + bbox = ccd0.amp_geom.serial_overscan + bbox.grow(-buffer) for slot in slots: - ccd = sensorTest.MaskedCCD(bias_files[slot]) - if bbox is None: - bbox = ccd.amp_geom.serial_overscan - bbox.grow(-buffer) - for amp in ccd: - image = ccd[amp].getImage() - overscans.append(image.Factory(image, bbox).getArray()) + if slot not in bias_files: + for amp in ccd0: + overscans.append(np.zeros((bbox.getHeight(), bbox.getWidth()))) + else: + ccd = sensorTest.MaskedCCD(bias_files[slot]) + for amp in ccd: + image = ccd[amp].getImage() + overscans.append(image.Factory(image, bbox).getArray()) namps = len(overscans) data = np.array([np.corrcoef(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1] From 0838b82281b3b94e9540ffabc302ea1bbb64bc1f Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 5 Oct 2019 22:56:06 -0700 Subject: [PATCH 2/6] skip raft mosaics for corner rafts; suppress parsl import warning; handle temp names for mask files using tempfile; enable setting of max_job_time in lcatr.cfg --- .../v0/producer_raft_results_summary_BOT.py | 3 ++- .../raft_results_summary_BOT/v0/raft_results_task.py | 11 ++++++++--- python/multiprocessor_execution.py | 6 ++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/harnessed_jobs/raft_results_summary_BOT/v0/producer_raft_results_summary_BOT.py b/harnessed_jobs/raft_results_summary_BOT/v0/producer_raft_results_summary_BOT.py index f5f8483..1e981d6 100755 --- a/harnessed_jobs/raft_results_summary_BOT/v0/producer_raft_results_summary_BOT.py +++ b/harnessed_jobs/raft_results_summary_BOT/v0/producer_raft_results_summary_BOT.py @@ -79,7 +79,8 @@ def make_focal_plane_plots(): = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', 'raft_results_summary_BOT', 'v0', 'raft_results_task.py') - installed_rafts = camera_info.get_installed_raft_names() +# installed_rafts = camera_info.get_installed_raft_names() + installed_rafts = camera_info.installed_science_rafts run_python_task_or_cl_script(raft_results_task, raft_results_task_script, device_names=installed_rafts) diff --git a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py index 0a84b21..af06f18 100755 --- a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py +++ b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py @@ -32,11 +32,16 @@ def plt_savefig(filename): # compliance. Use one of the median bias files for this since they # should be available no matter which analysis tasks are run. bias_frames = get_raft_files_by_slot(raft_name, 'median_bias.fits') - mask_files = get_raft_files_by_slot(raft_name, 'edge_rolloff_mask.fits') - + try: + mask_files = get_raft_files_by_slot(raft_name, + 'edge_rolloff_mask.fits') + except FileNotFoundError: + input_mask = None + else: + input_mask = list(mask_files.values())[0] total_num, rolloff_mask \ = sensorTest.pixel_counts(list(bias_frames.values())[0], - input_mask=list(mask_files.values())[0]) + input_mask=input_mask) # Exposure time (in seconds) for 95th percentile dark current shot # noise calculation. diff --git a/python/multiprocessor_execution.py b/python/multiprocessor_execution.py index 192f090..3e19303 100644 --- a/python/multiprocessor_execution.py +++ b/python/multiprocessor_execution.py @@ -12,7 +12,7 @@ from parsl_execution import parsl_sensor_analyses, \ parsl_device_analysis_pool except ImportError as eobj: - warnings.warn(f'ImportError: {eobj}') + #warnings.warn(f'ImportError: {eobj}') pass from ssh_dispatcher import ssh_device_analysis_pool @@ -79,7 +79,9 @@ def run_device_analysis_pool(task_func, device_names, processes=None, cwd=None, processes=processes, cwd=cwd) if siteUtils.getUnitType() == 'LCA-10134_Cryostat': - return ssh_device_analysis_pool(task_func, device_names, cwd=cwd) + max_time = os.environ.get('LCATR_MAX_JOB_TIME', None) + return ssh_device_analysis_pool(task_func, device_names, cwd=cwd, + max_time=max_time) if processes is None: # Use the maximum number of cores available, reserving one for From 826e6df98a5bece4a2a4790d210cf3690b24eb08 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 11 Oct 2019 12:57:38 -0700 Subject: [PATCH 3/6] use \w to match any alphanumeric character for the REB slot since corner raft detectors have names like R00_SW0. --- python/bot_eo_analyses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index 72dc549..e3249df 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -209,7 +209,7 @@ def __call__(self, file_pattern): if self.run is None: return _get_amplifier_gains(file_pattern) # Extract the det_name from the file pattern. - match = re.search('R\d\d_S\d\d', file_pattern) + match = re.search('R\d\d_S\w\d', file_pattern) if match is None: message = f"no det_name match in {file_pattern}" raise RuntimeError("GetAmplifierGains.__call__: " + message) From a74f249fa384e532faeb17358897e549469d4cc6 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 11 Oct 2019 13:58:29 -0700 Subject: [PATCH 4/6] No need for error-handling in this script for finding gains; get_amplifier_gains will resolve any errors --- harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py b/harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py index 77303a4..b4e63a4 100755 --- a/harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py +++ b/harnessed_jobs/flat_pairs_BOT/v0/flat_pairs_jh_task.py @@ -22,14 +22,7 @@ def flat_pairs_jh_task(det_name): return None mask_files = sorted(glob.glob('{}_*mask.fits'.format(file_prefix))) - try: - eotest_results_file \ - = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), - jobname='fe55_analysis_BOT')[0] - except IndexError: - print("flat_pairs_jh_task: Fe55 eotest results file not found for ", - file_prefix, ". Using unit gains.") - eotest_results_file = None + eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(file_prefix) From 42eedfaa4fe0838de076311406b79066274ac2be Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 11 Oct 2019 14:29:55 -0700 Subject: [PATCH 5/6] handle case where Fe55 gains are obtained from a previous run and the Fe55 analysis job is not run --- harnessed_jobs/cti_BOT/v0/cte_jh_task.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/harnessed_jobs/cti_BOT/v0/cte_jh_task.py b/harnessed_jobs/cti_BOT/v0/cte_jh_task.py index 1e4abba..93bfdca 100755 --- a/harnessed_jobs/cti_BOT/v0/cte_jh_task.py +++ b/harnessed_jobs/cti_BOT/v0/cte_jh_task.py @@ -29,9 +29,13 @@ def cte_jh_task(det_name): eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) - results_file = siteUtils.dependency_glob(eotest_results_file, - jobname='fe55_analysis_BOT')[0] - shutil.copy(results_file, eotest_results_file) + try: + results_file = siteUtils.dependency_glob(eotest_results_file, + jobname='fe55_analysis_BOT')[0] + except IndexError: + pass + else: + shutil.copy(results_file, eotest_results_file) bias_frame = bias_filename(file_prefix) From 1f2dfaa3ad9db8b75e8022535e33781d3320d82f Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 11 Oct 2019 21:40:40 -0700 Subject: [PATCH 6/6] handle case where bot_eo config omits biasnoise analysis --- .../raft_results_summary_BOT/v0/raft_results_task.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py index af06f18..69f5464 100755 --- a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py +++ b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py @@ -60,7 +60,11 @@ def plt_savefig(filename): eotest_results.add_seg_result(amp, 'MAX_FRAC_DEV', 0.) eotest_results.add_seg_result(amp, 'DC95_SHOT_NOISE', np.float(shot_noise[i])) - eotest_results['TOTAL_NOISE'][i] = total_noise[i] + try: + eotest_results['TOTAL_NOISE'][i] = total_noise[i] + except KeyError: + eotest_results.add_seg_result(amp, 'TOTAL_NOISE', + np.float(total_noise[i])) eotest_results.write(filename) run = siteUtils.getRunNumber()