From 3b043d2fbc5b6c914e6fbc54021b4e649ec60f2c Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 19 Feb 2021 22:24:39 -0800 Subject: [PATCH] add pca_superbias to bias frame task --- data/BOT_jh_glob_patterns.ini | 2 +- .../bias_frame_BOT/v0/bias_frame_jh_task.py | 12 +++++++++--- python/bot_eo_analyses.py | 9 +++++++++ python/bot_eo_validators.py | 4 +++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/data/BOT_jh_glob_patterns.ini b/data/BOT_jh_glob_patterns.ini index 802ad3c..e0c2fef 100644 --- a/data/BOT_jh_glob_patterns.ini +++ b/data/BOT_jh_glob_patterns.ini @@ -1,7 +1,7 @@ [BOT_acq] fe55 = fe55_flat_*_flat_* fe55_bias = fe55_flat_bias_* -bias_frame = bias_bias_* +bias_frame = *_bias_* bias_stability = *_bias_* read_noise = bias_bias_* raft_noise_correlations = bias_bias_* diff --git a/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py b/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py index c2636fc..274a083 100755 --- a/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py +++ b/harnessed_jobs/bias_frame_BOT/v0/bias_frame_jh_task.py @@ -8,14 +8,17 @@ def bias_frame_jh_task(det_name): import siteUtils import json from bot_eo_analyses import glob_pattern, bias_frame_task, \ - bias_stability_task + bias_stability_task, pca_corrected_superbias run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('bias_frame', det_name), acq_jobname=acq_jobname, - description='Bias frames:') + description='Bias frames:')[10:] + # Skip the first 10 bias files to avoid transient features in the + # imaging array from after just starting the run. + bias_files = sorted(bias_files, key=os.path.basename)[5:] if not bias_files: print("bias_frame_task: Needed data files are missing for detector", det_name) @@ -30,7 +33,7 @@ def bias_frame_jh_task(det_name): det_name) return None - bias_frame, pca_files = bias_frame_task(run, det_name, bias_files[1:]) + bias_frame, pca_files = bias_frame_task(run, det_name, bias_files) bias_stability_files = sorted(bias_stability_files) if not os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True': @@ -39,6 +42,9 @@ def bias_frame_jh_task(det_name): bias_stability_task(run, det_name, bias_stability_files, pca_files=pca_files) + if pca_files is not None: + pca_corrected_superbias(run, det_name, bias_files, pca_files) + return bias_frame diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index 2b3d303..f7a8967 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -45,6 +45,7 @@ 'get_analysis_types', 'fe55_task', 'bias_frame_task', + 'pca_corrected_superbias', 'bias_stability_task', 'scan_mode_analysis_task', 'get_scan_mode_files', @@ -564,6 +565,13 @@ def bias_frame_task(run, det_name, bias_files, bias_frame=None): return bias_frame, pca_files + +def pca_corrected_superbias(run, det_name, bias_files, pca_bias_files): + file_prefix = make_file_prefix(run, det_name) + outfile = f'{file_prefix}_pca_superbias.fits' + sensorTest.pca_superbias(bias_files, pca_bias_files, outfile) + + def image_stats(image, nsigma=10): """Compute clipped mean and stdev of the image.""" stat_ctrl = afwMath.StatisticsControl(numSigmaClip=nsigma) @@ -571,6 +579,7 @@ def image_stats(image, nsigma=10): stats = afwMath.makeStatistics(image, flags=flags, sctrl=stat_ctrl) return stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) + def bias_stability_task(run, det_name, bias_files, nsigma=10, pca_files=None): """Compute amp-wise bias stability time histories and serial profiles.""" diff --git a/python/bot_eo_validators.py b/python/bot_eo_validators.py index 5bbd593..988efd8 100644 --- a/python/bot_eo_validators.py +++ b/python/bot_eo_validators.py @@ -72,9 +72,11 @@ def validate_bias_frame(results, det_names): bias_frame = f'{file_prefix}_median_bias.fits' rolloff_mask = f'{file_prefix}_edge_rolloff_mask.fits' pca_bias_file = f'{file_prefix}_pca_bias.fits' + pca_superbias = f'{file_prefix}_pca_superbias.fits' # Add/update the metadata to the primary HDU of these files. - for fitsfile in (bias_frame, rolloff_mask, pca_bias_file): + for fitsfile in (bias_frame, rolloff_mask, pca_bias_file, + pca_superbias): if os.path.isfile(fitsfile): eotestUtils.addHeaderData(fitsfile, TESTTYPE='BIAS', DATE=eotestUtils.utc_now_isoformat())