From c6aa698f113e13ed57105d4b53125a9eee5c5fbe Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 11 Jun 2022 17:05:40 -0700 Subject: [PATCH 1/3] superbias handling --- .../bias_frame_BOT/v0/bias_frame_jh_task.py | 7 ++++--- .../v0/raft_results_task.py | 3 ++- python/bot_eo_analyses.py | 13 ++++++++++--- python/stage_bot_data.py | 3 +++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py b/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py index 3cb114c..913b34c 100755 --- a/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py +++ b/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py @@ -8,7 +8,7 @@ def bias_frame_jh_task(det_name): import siteUtils import json from bot_eo_analyses import glob_pattern, bias_frame_task, \ - bias_stability_task, pca_corrected_superbias + make_bias_filename, bias_stability_task, pca_corrected_superbias run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') @@ -39,12 +39,13 @@ def bias_frame_jh_task(det_name): if not os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True': pca_files = None if siteUtils.get_analysis_run('bias') == 'rowcol': - pca_files = 'rowcol' + bias_frame = make_bias_filename(run, det_name) + pca_files = 'rowcol', bias_frame print("pca_files:", pca_files) bias_stability_task(run, det_name, bias_stability_files, pca_files=pca_files) - if pca_files is not None and pca_files != 'rowcol': + if pca_files is not None and pca_files[0] != 'rowcol': pca_corrected_superbias(run, det_name, bias_files, pca_files) return bias_frame diff --git a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py index d968c82..13133a2 100755 --- a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py +++ b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py @@ -114,7 +114,7 @@ def plt_savefig(filename): bias_run = siteUtils.get_analysis_run('bias') if bias_run is not None and bias_run.lower() == 'rowcol': for key in bias_frames: - bias_frames[key] = 'rowcol' + bias_frames[key] = ('rowcol', bias_frames[key]) # Dark mosaic dark_files = None @@ -127,6 +127,7 @@ def plt_savefig(filename): except FileNotFoundError as eobj: print(eobj) if dark_files is not None: + print('raft_results_mosaic:', bias_frames) dark_mosaic = raftTest.make_raft_mosaic(dark_files, gains=gains, bias_frames=bias_frames) dark_mosaic.plot(title=f'{title}, medianed dark frames', diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index df60748..cdb5809 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -37,6 +37,7 @@ __all__ = ['make_file_prefix', 'make_rolloff_mask', + 'make_bias_filename', 'append_acq_run', 'make_title', 'glob_pattern', @@ -305,9 +306,15 @@ def bias_filename(run, det_name): bias_run = siteUtils.get_analysis_run('bias') if bias_run is not None and bias_run.lower() == 'rowcol': # This will set the bias_frame option for all tasks to - # 'rowcol' and so the eotest.sensor.MaskedCCD code will use - # the parallel+serial overscan model for bias subtraction. - return 'rowcol' + # ('rowcol', superbias_file) and so the eotest.sensor.MaskedCCD + # code will use the parallel+serial overscan correction and + # superbias subtraction for bias correction. + superbias_file = make_bias_filename(run, det_name) + bias_frame = siteUtils.dependency_glob( + superbias_file, description='superbias file:') + bias_frame = bias_frame[0] if bias_frame else None + print('bias_frame:', bias_frame) + return 'rowcol', bias_frame elif bias_run is None: if use_pca_bias: file_prefix = make_file_prefix(run, det_name) diff --git a/python/stage_bot_data.py b/python/stage_bot_data.py index ad6929d..127979b 100644 --- a/python/stage_bot_data.py +++ b/python/stage_bot_data.py @@ -98,6 +98,9 @@ def get_isr_files(det_name, run): if isinstance(bias_fn, str): if os.path.isfile(bias_fn): files.add(bias_fn) + elif isinstance(bias_fn, (tuple, list)) and bias_fn[0] == 'rowcol': + if bias_fn[1] is not None: + files.add(bias_fn[1]) else: files = files.union(bias_fn) except IndexError: From f29c11e07d739dc86c46605c87c284b919cf7149 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sun, 12 Jun 2022 12:32:42 -0700 Subject: [PATCH 2/3] remove debugging print statements --- harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py | 1 - python/bot_eo_analyses.py | 1 - 2 files changed, 2 deletions(-) diff --git a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py index 13133a2..ccc771f 100755 --- a/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py +++ b/harnessed_jobs/raft_results_summary_BOT/v0/raft_results_task.py @@ -127,7 +127,6 @@ def plt_savefig(filename): except FileNotFoundError as eobj: print(eobj) if dark_files is not None: - print('raft_results_mosaic:', bias_frames) dark_mosaic = raftTest.make_raft_mosaic(dark_files, gains=gains, bias_frames=bias_frames) dark_mosaic.plot(title=f'{title}, medianed dark frames', diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index cdb5809..a194162 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -313,7 +313,6 @@ def bias_filename(run, det_name): bias_frame = siteUtils.dependency_glob( superbias_file, description='superbias file:') bias_frame = bias_frame[0] if bias_frame else None - print('bias_frame:', bias_frame) return 'rowcol', bias_frame elif bias_run is None: if use_pca_bias: From 9b89c3c7627d2a843274103202961f816364c0fe Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sun, 12 Jun 2022 16:18:15 -0700 Subject: [PATCH 3/3] use more appropriate local variable names to describe bias model used --- .../bias_frame_BOT/v0/bias_frame_jh_task.py | 22 ++++++++++--------- python/bot_eo_analyses.py | 5 ++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py b/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py index 913b34c..e211919 100755 --- a/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py +++ b/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py @@ -8,7 +8,7 @@ def bias_frame_jh_task(det_name): import siteUtils import json from bot_eo_analyses import glob_pattern, bias_frame_task, \ - make_bias_filename, bias_stability_task, pca_corrected_superbias + bias_stability_task, pca_corrected_superbias run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') @@ -33,22 +33,24 @@ def bias_frame_jh_task(det_name): det_name) return None - bias_frame, pca_files = bias_frame_task(run, det_name, bias_files) + superbias_file, bias_model_components \ + = bias_frame_task(run, det_name, bias_files) bias_stability_files = sorted(bias_stability_files) if not os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True': - pca_files = None + bias_model_components = None if siteUtils.get_analysis_run('bias') == 'rowcol': - bias_frame = make_bias_filename(run, det_name) - pca_files = 'rowcol', bias_frame - print("pca_files:", pca_files) + bias_model_components = 'rowcol', superbias_file + print("bias_model_components:", bias_model_components) bias_stability_task(run, det_name, bias_stability_files, - pca_files=pca_files) + bias_model_components=bias_model_components) - if pca_files is not None and pca_files[0] != 'rowcol': - pca_corrected_superbias(run, det_name, bias_files, pca_files) + if (bias_model_components is not None and + bias_model_components[0] != 'rowcol'): + pca_corrected_superbias(run, det_name, bias_files, + bias_model_components) - return bias_frame + return superbias_file if __name__ == '__main__': diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index a194162..e27b379 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -37,7 +37,6 @@ __all__ = ['make_file_prefix', 'make_rolloff_mask', - 'make_bias_filename', 'append_acq_run', 'make_title', 'glob_pattern', @@ -696,7 +695,7 @@ def image_stats(image, nsigma=10): def bias_stability_task(run, det_name, bias_files, nsigma=10, - pca_files=None, llc_size=200): + bias_model_components=None, llc_size=200): """Compute amp-wise bias stability time histories and serial profiles.""" raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) @@ -714,7 +713,7 @@ def bias_stability_task(run, det_name, bias_files, nsigma=10, key = f'TEMP{i}' if key in hdus['REB_COND'].header: temps[key] = hdus['REB_COND'].header[key] - ccd = sensorTest.MaskedCCD(bias_file, bias_frame=pca_files) + ccd = sensorTest.MaskedCCD(bias_file, bias_frame=bias_model_components) for amp in ccd: # Retrieve the per row overscan subtracted imaging section. amp_image = ccd.unbiased_and_trimmed_image(amp)