From b8580ab9588c3430e786fd43def0cef9515726e9 Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Sun, 14 May 2017 18:01:42 -0700 Subject: [PATCH 1/7] change plots from formerly requested bar charts to now requested point plots --- .../v0/producer_collect_raft_results.py | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py index 4748f99..7fc681a 100755 --- a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py +++ b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py @@ -6,6 +6,7 @@ from __future__ import print_function import os from collections import OrderedDict +import numpy as np import matplotlib.pyplot as plt import lsst.eotest.sensor as sensorTest import lsst.eotest.raft as raftTest @@ -25,6 +26,10 @@ def slot_dependency_glob(pattern, jobname): bias_files = slot_dependency_glob('*_mean_bias_*.fits', 'fe55_raft_analysis') total_num, rolloff_mask = sensorTest.pixel_counts(bias_files.values()[0]) +# Exposure time (in seconds) for 95th percentile dark current shot +# noise calculation. +exptime = 16. + raft_id = siteUtils.getUnitId() raft = camera_components.Raft.create_from_etrav(raft_id) summary_files = dependency_glob('summary.lims') @@ -38,6 +43,16 @@ def slot_dependency_glob(pattern, jobname): rolloff_mask) repackager.process_files(summary_files, sensor_id=sensor_id) + # Add 95th percentile dark current shot noise and add in quadrature + # to read noise to produce updated total noise. + shot_noise = repackager.eotest_results['DARK_CURRENT_95']*exptime + total_noise = np.sqrt(repackager.eotest_results['READ_NOISE']**2 + + shot_noise**2) + for i, amp in repackager.eotest_results['AMP']: + repackager.eotest_results.add_seg_result(amp, 'DC95_SHOT_NOISE', + np.float(shot_noise[i])) + repackager.eotest_results['TOTAL_NOISE'][i] = total_noise[i] + outfile = '%s_eotest_results.fits' % sensor_id repackager.write(outfile) results_files[slot] = outfile @@ -87,33 +102,37 @@ def slot_dependency_glob(pattern, jobname): print(files) print(eobj) -# Raft-level bar charts of read noise, nonlinearity, serial and parallel CTI, +# Raft-level plots of read noise, nonlinearity, serial and parallel CTI, # charge diffusion PSF, and gains from Fe55 and PTC. -bar_charts = raftTest.RaftBarCharts(results_files) +spec_plots = raftTest.RaftSpecPlots(results_files) + +# Add 95th percentile dark current shot noise for 16 second exposure for +# total noise plots. +spec_plots.results = raftTest.add_shot_noise(spec_plots.results, exptime=16) -bar_charts.make_bar_chart('READ_NOISE', 'read noise (-e rms)', spec=9, - title=raft_id) -plt.savefig('%s_read_noise.png' % raft_id) +columns = 'READ_NOISE DC95_SHOT_NOISE TOTAL_NOISE'.split() +spec_plots.make_multi_column_plot(columns, 'noise per pixel (-e rms)', spec=9, + title=raft_id) +plt.savefig('%s_total_noise.png' % raft_id) -bar_charts.make_bar_chart('MAX_FRAC_DEV', - 'non-linearity (max. fractional deviation)', - spec=0.02, title=raft_id) +spec_plots.make_plot('MAX_FRAC_DEV', + 'non-linearity (max. fractional deviation)', + spec=0.02, title=raft_id) plt.savefig('%s_linearity.png' % raft_id) -bar_charts.make_multi_bar_chart(('CTI_LOW_SERIAL', 'CTI_HIGH_SERIAL'), - 'Serial CTI', spec=5e-6, ylog=True, - title=raft_id, colors='br') +spec_plots.make_multi_column_plot(('CTI_LOW_SERIAL', 'CTI_HIGH_SERIAL'), + 'Serial CTI', spec=5e-6, ylog=True, + title=raft_id, colors='br') plt.savefig('%s_serial_cti.png' % raft_id) -bar_charts.make_multi_bar_chart(('CTI_LOW_PARALLEL', 'CTI_HIGH_PARALLEL'), - 'Parallel CTI', spec=3e-6, ylog=True, - title=raft_id, colors='br') +spec_plots.make_multi_column_plot(('CTI_LOW_PARALLEL', 'CTI_HIGH_PARALLEL'), + 'Parallel CTI', spec=3e-6, ylog=True, + title=raft_id, colors='br') plt.savefig('%s_parallel_cti.png' % raft_id) -bar_charts.make_bar_chart('PSF_SIGMA', 'PSF sigma (microns)', spec=5., - title=raft_id) +spec_plots.make_plot('PSF_SIGMA', 'PSF sigma (microns)', spec=5., title=raft_id) plt.savefig('%s_psf_sigma.png' % raft_id) -bar_charts.make_multi_bar_chart(('GAIN', 'PTC_GAIN'), 'System Gain (e-/ADU)', - title=raft_id, colors='br', ylog=True) +spec_plots.make_multi_column_plot(('GAIN', 'PTC_GAIN'), 'System Gain (e-/ADU)', + title=raft_id, colors='br', ylog=True) plt.savefig('%s_system_gain.png' % raft_id) From 252fa23c19fe62a4e4cde59f678f44aa98147201 Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Sun, 14 May 2017 21:50:10 -0700 Subject: [PATCH 2/7] ensure linearity plots use 1e4-9e4 range and max frac dev of 0.03 --- .../v0/producer_collect_raft_results.py | 9 +++++++-- .../v0/producer_flat_pairs_raft_analysis.py | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py index 7fc681a..704bb64 100755 --- a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py +++ b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py @@ -21,6 +21,8 @@ def slot_dependency_glob(pattern, jobname): jobname=jobname)) return OrderedDict([(fn.split('/')[-2], fn) for fn in files]) +run_number = siteUtils.getRunNumber() + # Use a mean bias file to determine the maximum number of active # pixels for the image quality statistics. bias_files = slot_dependency_glob('*_mean_bias_*.fits', 'fe55_raft_analysis') @@ -121,12 +123,14 @@ def slot_dependency_glob(pattern, jobname): plt.savefig('%s_linearity.png' % raft_id) spec_plots.make_multi_column_plot(('CTI_LOW_SERIAL', 'CTI_HIGH_SERIAL'), - 'Serial CTI', spec=5e-6, ylog=True, + 'Serial CTI (ppm)', spec=5e-6, + yscaling=1e6, yerrors=True, title=raft_id, colors='br') plt.savefig('%s_serial_cti.png' % raft_id) spec_plots.make_multi_column_plot(('CTI_LOW_PARALLEL', 'CTI_HIGH_PARALLEL'), - 'Parallel CTI', spec=3e-6, ylog=True, + 'Parallel CTI (ppm)', spec=3e-6, + yscaling=1e6, yerrors=True, title=raft_id, colors='br') plt.savefig('%s_parallel_cti.png' % raft_id) @@ -134,5 +138,6 @@ def slot_dependency_glob(pattern, jobname): plt.savefig('%s_psf_sigma.png' % raft_id) spec_plots.make_multi_column_plot(('GAIN', 'PTC_GAIN'), 'System Gain (e-/ADU)', + yerrors=True, title=raft_id, colors='br', ylog=True) plt.savefig('%s_system_gain.png' % raft_id) diff --git a/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py b/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py index b5b4a6b..62ca824 100755 --- a/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py +++ b/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py @@ -17,9 +17,10 @@ def run_flat_pair_task(sensor_id): gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', sensor_id=sensor_id) + use_exptime = True task = sensorTest.FlatPairTask() task.run(sensor_id, flat_files, mask_files, gains, - linearity_spec_range=(1e4, 9e4)) + linearity_spec_range=(1e4, 9e4), use_exptime=use_exptime) results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) @@ -27,10 +28,12 @@ def run_flat_pair_task(sensor_id): detresp_file = '%s_det_response.fits' % sensor_id siteUtils.make_png_file(plots.linearity, '%s_linearity.png' % sensor_id, - detresp_file=detresp_file) + detresp_file=detresp_file, max_dev=0.03, + use_exptime=use_exptime) siteUtils.make_png_file(plots.linearity_resids, '%s_linearity_resids.png' % sensor_id, - detresp_file=detresp_file) + detresp_file=detresp_file, max_dev=0.03, + Ne_bounds=(1e4, 9e4), use_exptime=use_exptime) if __name__ == '__main__': sensor_analyses(run_flat_pair_task) From dcfead55d795d34da28ab4a638a1b0f5566cebc8 Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Tue, 16 May 2017 13:10:47 -0700 Subject: [PATCH 3/7] enable linearity spec range to be set by clients for plots; add annotation to mosaics --- .../v0/producer_collect_raft_results.py | 68 +++++++++++-------- .../v0/producer_flat_pairs_raft_analysis.py | 6 +- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py index 704bb64..49a2e61 100755 --- a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py +++ b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py @@ -50,7 +50,7 @@ def slot_dependency_glob(pattern, jobname): shot_noise = repackager.eotest_results['DARK_CURRENT_95']*exptime total_noise = np.sqrt(repackager.eotest_results['READ_NOISE']**2 + shot_noise**2) - for i, amp in repackager.eotest_results['AMP']: + for i, amp in enumerate(repackager.eotest_results['AMP']): repackager.eotest_results.add_seg_result(amp, 'DC95_SHOT_NOISE', np.float(shot_noise[i])) repackager.eotest_results['TOTAL_NOISE'][i] = total_noise[i] @@ -65,28 +65,34 @@ def slot_dependency_glob(pattern, jobname): gains[slot] = dict([(amp, gain) for amp, gain in zip(results['AMP'], results['GAIN'])]) +title = '%s, %s' % (raft_id, run_number) +file_prefix = '%s_%s' % (raft_id, run_number) + # Raft-level mosaics of median darks, bias, superflats high and low. dark_mosaic = raftTest.RaftMosaic(slot_dependency_glob('*median_dark_bp.fits', 'bright_defects_raft'), gains=gains) -dark_mosaic.plot(title='%s, medianed dark frames' % raft_id) -plt.savefig('%s_medianed_dark.png' % raft_id) +dark_mosaic.plot(title='%s, medianed dark frames' % title, + annotation='e-/pixel, gain-corrected, bias-subtracted') +plt.savefig('%s_medianed_dark.png' % file_prefix) del dark_mosaic mean_bias = raftTest.RaftMosaic(bias_files) -mean_bias.plot(title='%s, mean bias frames' % raft_id) -plt.savefig('%s_mean_bias.png' % raft_id) +mean_bias.plot(title='%s, mean bias frames' % title, annotation='ADU/pixel') +plt.savefig('%s_mean_bias.png' % file_prefix) del mean_bias sflat_high = raftTest.RaftMosaic(slot_dependency_glob('*superflat_high.fits', 'cte_raft'), gains=gains) -sflat_high.plot(title='%s, high flux superflat' % raft_id) +sflat_high.plot(title='%s, high flux superflat' % title, + annotation='e-/pixel, gain-corrected, bias-subtracted') plt.savefig('%s_superflat_high.png' % raft_id) del sflat_high sflat_low = raftTest.RaftMosaic(slot_dependency_glob('*superflat_low.fits', 'cte_raft'), gains=gains) -sflat_low.plot(title='%s, low flux superflat' % raft_id) +sflat_low.plot(title='%s, low flux superflat' % title, + annotation='e-/pixel, gain-corrected, bias-subtracted') plt.savefig('%s_superflat_low.png' % raft_id) del sflat_low @@ -97,8 +103,9 @@ def slot_dependency_glob(pattern, jobname): siteUtils.getProcessName('qe_raft_acq')) try: flat = raftTest.RaftMosaic(files, gains=gains) - flat.plot(title='%s, %i nm' % (raft_id, wl)) - plt.savefig('%s_%04inm_flat.png' % (raft_id, wl)) + flat.plot(title='%s, %i nm' % (title, wl), + annotation='e-/pixel, gain-corrected, bias-subtracted') + plt.savefig('%s_%04inm_flat.png' % (file_prefix, wl)) del flat except IndexError as eobj: print(files) @@ -108,36 +115,39 @@ def slot_dependency_glob(pattern, jobname): # charge diffusion PSF, and gains from Fe55 and PTC. spec_plots = raftTest.RaftSpecPlots(results_files) -# Add 95th percentile dark current shot noise for 16 second exposure for -# total noise plots. -spec_plots.results = raftTest.add_shot_noise(spec_plots.results, exptime=16) - columns = 'READ_NOISE DC95_SHOT_NOISE TOTAL_NOISE'.split() spec_plots.make_multi_column_plot(columns, 'noise per pixel (-e rms)', spec=9, - title=raft_id) -plt.savefig('%s_total_noise.png' % raft_id) + title=title) +plt.savefig('%s_total_noise.png' % file_prefix) spec_plots.make_plot('MAX_FRAC_DEV', 'non-linearity (max. fractional deviation)', - spec=0.02, title=raft_id) -plt.savefig('%s_linearity.png' % raft_id) + spec=0.03, title=title) +plt.savefig('%s_linearity.png' % file_prefix) spec_plots.make_multi_column_plot(('CTI_LOW_SERIAL', 'CTI_HIGH_SERIAL'), - 'Serial CTI (ppm)', spec=5e-6, - yscaling=1e6, yerrors=True, - title=raft_id, colors='br') -plt.savefig('%s_serial_cti.png' % raft_id) + 'Serial CTI (ppm)', spec=(5e-6, 3e-5), + title=title, yscaling=1e6, yerrors=True, + colors='br', ymax=4e-5) +plt.savefig('%s_serial_cti.png' % file_prefix) spec_plots.make_multi_column_plot(('CTI_LOW_PARALLEL', 'CTI_HIGH_PARALLEL'), 'Parallel CTI (ppm)', spec=3e-6, - yscaling=1e6, yerrors=True, - title=raft_id, colors='br') -plt.savefig('%s_parallel_cti.png' % raft_id) + title=title, yscaling=1e6, yerrors=True, + colors='br') +plt.savefig('%s_parallel_cti.png' % file_prefix) -spec_plots.make_plot('PSF_SIGMA', 'PSF sigma (microns)', spec=5., title=raft_id) -plt.savefig('%s_psf_sigma.png' % raft_id) +spec_plots.make_plot('PSF_SIGMA', 'PSF sigma (microns)', spec=5., title=title, + ymax=5.2) +plt.savefig('%s_psf_sigma.png' % file_prefix) spec_plots.make_multi_column_plot(('GAIN', 'PTC_GAIN'), 'System Gain (e-/ADU)', - yerrors=True, - title=raft_id, colors='br', ylog=True) -plt.savefig('%s_system_gain.png' % raft_id) + yerrors=True, title=title, colors='br') +plt.savefig('%s_system_gain.png' % file_prefix) + +spec_plots.make_plot('DARK_CURRENT_95', + '95th percentile dark current (e-/pixel/s)', + spec=0.2, title=title) +plt.savefig('%s_dark_current.png' % file_prefix) + +raise RuntimeError("Raise an exception to enable a retry in eT.") diff --git a/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py b/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py index 62ca824..ae298f7 100755 --- a/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py +++ b/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py @@ -25,15 +25,17 @@ def run_flat_pair_task(sensor_id): results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) + Ne_bounds = (1e4, 9e4) + detresp_file = '%s_det_response.fits' % sensor_id siteUtils.make_png_file(plots.linearity, '%s_linearity.png' % sensor_id, detresp_file=detresp_file, max_dev=0.03, - use_exptime=use_exptime) + use_exptime=use_exptime, Ne_bounds=Ne_bounds) siteUtils.make_png_file(plots.linearity_resids, '%s_linearity_resids.png' % sensor_id, detresp_file=detresp_file, max_dev=0.03, - Ne_bounds=(1e4, 9e4), use_exptime=use_exptime) + Ne_bounds=Ne_bounds, use_exptime=use_exptime) if __name__ == '__main__': sensor_analyses(run_flat_pair_task) From b1e2ff5611b4f89f9bae068b51e8a38af8a73989 Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Tue, 16 May 2017 16:14:12 -0700 Subject: [PATCH 4/7] don't do overscan subtraction in mean bias frame mosaic; add QE summary plot --- .../v0/producer_collect_raft_results.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py index 49a2e61..82c6879 100755 --- a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py +++ b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py @@ -77,7 +77,7 @@ def slot_dependency_glob(pattern, jobname): plt.savefig('%s_medianed_dark.png' % file_prefix) del dark_mosaic -mean_bias = raftTest.RaftMosaic(bias_files) +mean_bias = raftTest.RaftMosaic(bias_files, bias_subtract=False) mean_bias.plot(title='%s, mean bias frames' % title, annotation='ADU/pixel') plt.savefig('%s_mean_bias.png' % file_prefix) del mean_bias @@ -111,6 +111,13 @@ def slot_dependency_glob(pattern, jobname): print(files) print(eobj) +# QE summary plot. +qe_summary_lims = \ + siteUtils.dependency_glob('summary.lims', jobname='qe_raft_analysis')[0] +qe_fig = raftTest.qe_summary_plot(qe_summary_lims, title=title) +plt.savefig('%s_QE_summary.png' % file_prefix) +del qe_fig + # Raft-level plots of read noise, nonlinearity, serial and parallel CTI, # charge diffusion PSF, and gains from Fe55 and PTC. spec_plots = raftTest.RaftSpecPlots(results_files) From 83847f016858f24e4ba5d7c74dfcd98dc1ed926a Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Wed, 17 May 2017 08:22:44 -0700 Subject: [PATCH 5/7] remove exception raising that enabled retries in eT --- .../collect_raft_results/v0/producer_collect_raft_results.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py index 82c6879..2f08ee8 100755 --- a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py +++ b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py @@ -156,5 +156,3 @@ def slot_dependency_glob(pattern, jobname): '95th percentile dark current (e-/pixel/s)', spec=0.2, title=title) plt.savefig('%s_dark_current.png' % file_prefix) - -raise RuntimeError("Raise an exception to enable a retry in eT.") From d119c468a667e5fe875717e0d7ae5461911bb05e Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Wed, 17 May 2017 22:02:14 -0700 Subject: [PATCH 6/7] add run number to superflat filename --- .../collect_raft_results/v0/producer_collect_raft_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py index 2f08ee8..2506304 100755 --- a/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py +++ b/harnessed_jobs/collect_raft_results/v0/producer_collect_raft_results.py @@ -86,14 +86,14 @@ def slot_dependency_glob(pattern, jobname): 'cte_raft'), gains=gains) sflat_high.plot(title='%s, high flux superflat' % title, annotation='e-/pixel, gain-corrected, bias-subtracted') -plt.savefig('%s_superflat_high.png' % raft_id) +plt.savefig('%s_superflat_high.png' % file_prefix) del sflat_high sflat_low = raftTest.RaftMosaic(slot_dependency_glob('*superflat_low.fits', 'cte_raft'), gains=gains) sflat_low.plot(title='%s, low flux superflat' % title, annotation='e-/pixel, gain-corrected, bias-subtracted') -plt.savefig('%s_superflat_low.png' % raft_id) +plt.savefig('%s_superflat_low.png' % file_prefix) del sflat_low # QE images at 350, 500, 620, 750, 870, and 1000 nm. From 031ea188ae7a55d243a7295a0b907240087ac880 Mon Sep 17 00:00:00 2001 From: Jim Chiang Date: Thu, 18 May 2017 15:07:53 -0700 Subject: [PATCH 7/7] insert run number in png filenames --- .../v0/producer_bright_defects_raft.py | 3 ++- .../cte_raft/v0/producer_cte_raft.py | 7 ++++--- .../v0/producer_dark_current_raft.py | 3 ++- .../v0/producer_dark_defects_raft.py | 3 ++- .../v0/producer_fe55_raft_analysis.py | 19 ++++++++++--------- .../v0/producer_flat_pairs_raft_analysis.py | 5 +++-- .../ptc_raft/v0/producer_ptc_raft.py | 3 ++- .../v0/producer_qa_plots_raft.py | 5 +++++ .../v0/producer_qe_raft_analysis.py | 3 ++- .../v0/producer_read_noise_raft.py | 3 ++- 10 files changed, 34 insertions(+), 20 deletions(-) diff --git a/harnessed_jobs/bright_defects_raft/v0/producer_bright_defects_raft.py b/harnessed_jobs/bright_defects_raft/v0/producer_bright_defects_raft.py index 50ae7d2..5ed62ad 100755 --- a/harnessed_jobs/bright_defects_raft/v0/producer_bright_defects_raft.py +++ b/harnessed_jobs/bright_defects_raft/v0/producer_bright_defects_raft.py @@ -9,6 +9,7 @@ from multiprocessor_execution import sensor_analyses def run_bright_pixels_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) dark_files = siteUtils.dependency_glob('S*/%s_dark_dark_*.fits' % sensor_id, jobname=siteUtils.getProcessName('dark_raft_acq'), description='Dark files:') @@ -22,7 +23,7 @@ def run_bright_pixels_task(sensor_id): task.run(sensor_id, dark_files, mask_files, gains) siteUtils.make_png_file(sensorTest.plot_flat, - '%s_medianed_dark.png' % sensor_id, + '%s_medianed_dark.png' % file_prefix, '%s_median_dark_bp.fits' % sensor_id, title='%s, medianed dark for bright defects analysis' % sensor_id) diff --git a/harnessed_jobs/cte_raft/v0/producer_cte_raft.py b/harnessed_jobs/cte_raft/v0/producer_cte_raft.py index d46ec25..07686cb 100755 --- a/harnessed_jobs/cte_raft/v0/producer_cte_raft.py +++ b/harnessed_jobs/cte_raft/v0/producer_cte_raft.py @@ -10,6 +10,7 @@ from multiprocessor_execution import sensor_analyses def run_cte_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) mask_files = \ eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id) gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis', @@ -49,18 +50,18 @@ def run_cte_task(sensor_id): if sflat_file.find('high') != -1: flux_level = 'high' siteUtils.make_png_file(sensorTest.plot_flat, - sflat_file.replace('.fits', '.png'), + sflat_file.replace('.fits', '.png').replace(sensor_id, file_prefix), sflat_file, title=('%s, CTE supeflat, %s flux ' % (sensor_id, flux_level))) siteUtils.make_png_file(plots.cte_profiles, ('%s_serial_oscan_%s.png' % - (sensor_id, flux_level)), + (file_prefix, flux_level)), flux_level, sflat_file, mask_files, serial=True) siteUtils.make_png_file(plots.cte_profiles, ('%s_parallel_oscan_%s.png' % - (sensor_id, flux_level)), + (file_prefix, flux_level)), flux_level, sflat_file, mask_files, serial=False) if __name__ == '__main__': diff --git a/harnessed_jobs/dark_current_raft/v0/producer_dark_current_raft.py b/harnessed_jobs/dark_current_raft/v0/producer_dark_current_raft.py index 9c3226a..f49e300 100755 --- a/harnessed_jobs/dark_current_raft/v0/producer_dark_current_raft.py +++ b/harnessed_jobs/dark_current_raft/v0/producer_dark_current_raft.py @@ -9,6 +9,7 @@ from multiprocessor_execution import sensor_analyses def run_dark_current_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) dark_files = siteUtils.dependency_glob('S*/%s_dark_dark_*.fits' % sensor_id, jobname=siteUtils.getProcessName('dark_raft_acq'), description='Dark files:') @@ -30,7 +31,7 @@ def run_dark_current_task(sensor_id): eo_results['TOTAL_NOISE'])) siteUtils.make_png_file(sensorTest.total_noise_histograms, - '%s_total_noise_hists.png' % sensor_id, + '%s_total_noise_hists.png' % file_prefix, dark_curr_pixels, read_noise, dark95s, exptime=16, title=sensor_id) diff --git a/harnessed_jobs/dark_defects_raft/v0/producer_dark_defects_raft.py b/harnessed_jobs/dark_defects_raft/v0/producer_dark_defects_raft.py index 574b498..7b9248a 100755 --- a/harnessed_jobs/dark_defects_raft/v0/producer_dark_defects_raft.py +++ b/harnessed_jobs/dark_defects_raft/v0/producer_dark_defects_raft.py @@ -9,6 +9,7 @@ from multiprocessor_execution import sensor_analyses def run_dark_pixels_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) sflat_files = siteUtils.dependency_glob('S*/%s_sflat_500_flat_H*.fits' % sensor_id, jobname=siteUtils.getProcessName('sflat_raft_acq'), description='Superflat files:') @@ -19,7 +20,7 @@ def run_dark_pixels_task(sensor_id): task.run(sensor_id, sflat_files, mask_files) siteUtils.make_png_file(sensorTest.plot_flat, - '%s_superflat_dark_defects.png' % sensor_id, + '%s_superflat_dark_defects.png' % file_prefix, '%s_median_sflat.fits' % sensor_id, title='%s, superflat for dark defects analysis' % sensor_id) diff --git a/harnessed_jobs/fe55_raft_analysis/v0/producer_fe55_raft_analysis.py b/harnessed_jobs/fe55_raft_analysis/v0/producer_fe55_raft_analysis.py index bba79a2..59e1a48 100755 --- a/harnessed_jobs/fe55_raft_analysis/v0/producer_fe55_raft_analysis.py +++ b/harnessed_jobs/fe55_raft_analysis/v0/producer_fe55_raft_analysis.py @@ -10,6 +10,7 @@ from multiprocessor_execution import sensor_analyses def run_fe55_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) fe55_files = siteUtils.dependency_glob('S*/%s_fe55_fe55_*.fits' % sensor_id, jobname=siteUtils.getProcessName('fe55_raft_acq'), description='Fe55 files:') @@ -26,7 +27,7 @@ def run_fe55_task(sensor_id): # print("processing fe55_zoom:", fe55_files[0]) siteUtils.make_png_file(sensorTest.fe55_zoom, - '%(sensor_id)s_fe55_zoom.png' % locals(), + '%(file_prefix)s_fe55_zoom.png' % locals(), fe55_files[0], size=250, amp=1) # @@ -36,18 +37,18 @@ def run_fe55_task(sensor_id): pixel_stats = sensorTest.Fe55PixelStats(fe55_files, sensor_id=sensor_id) siteUtils.make_png_file(pixel_stats.pixel_hists, - '%s_fe55_p3_p5_hists.png' % sensor_id, + '%s_fe55_p3_p5_hists.png' % file_prefix, pix0='p3', pix1='p5') siteUtils.make_png_file(pixel_stats.pixel_diff_profile, - '%s_fe55_p3_p5_profiles.png' % sensor_id, + '%s_fe55_p3_p5_profiles.png' % file_prefix, pixel_coord='x', pix0='p3', pix1='p5') siteUtils.make_png_file(pixel_stats.apflux_profile, - '%s_fe55_apflux_serial.png' % sensor_id) + '%s_fe55_apflux_serial.png' % file_prefix) siteUtils.make_png_file(pixel_stats.apflux_profile, - '%s_fe55_apflux_parallel.png' % sensor_id, + '%s_fe55_apflux_parallel.png' % file_prefix, pixel_coord='y') except Exception as eobj: @@ -69,20 +70,20 @@ def run_fe55_task(sensor_id): plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.gains, - '%s_gains.png' % sensor_id) + '%s_gains.png' % file_prefix) siteUtils.make_png_file(sensorTest.plot_flat, - '%s_mean_bias.png' % sensor_id, + '%s_mean_bias.png' % file_prefix, mean_bias_file, title='%s, mean bias frame' % sensor_id) fe55_file = glob.glob('%s_psf_results*.fits' % sensor_id)[0] siteUtils.make_png_file(plots.fe55_dists, - '%s_fe55_dists.png' % sensor_id, + '%s_fe55_dists.png' % file_prefix, fe55_file=fe55_file) siteUtils.make_png_file(plots.psf_dists, - '%s_psf_dists.png' % sensor_id, + '%s_psf_dists.png' % file_prefix, fe55_file=fe55_file) if __name__ == '__main__': diff --git a/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py b/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py index ae298f7..ba77b7f 100755 --- a/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py +++ b/harnessed_jobs/flat_pairs_raft_analysis/v0/producer_flat_pairs_raft_analysis.py @@ -9,6 +9,7 @@ from multiprocessor_execution import sensor_analyses def run_flat_pair_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) flat_files = siteUtils.dependency_glob('S*/%s_flat*flat?_*.fits' % sensor_id, jobname=siteUtils.getProcessName('flat_pair_raft_acq'), description='Flat files:') @@ -29,11 +30,11 @@ def run_flat_pair_task(sensor_id): detresp_file = '%s_det_response.fits' % sensor_id siteUtils.make_png_file(plots.linearity, - '%s_linearity.png' % sensor_id, + '%s_linearity.png' % file_prefix, detresp_file=detresp_file, max_dev=0.03, use_exptime=use_exptime, Ne_bounds=Ne_bounds) siteUtils.make_png_file(plots.linearity_resids, - '%s_linearity_resids.png' % sensor_id, + '%s_linearity_resids.png' % file_prefix, detresp_file=detresp_file, max_dev=0.03, Ne_bounds=Ne_bounds, use_exptime=use_exptime) diff --git a/harnessed_jobs/ptc_raft/v0/producer_ptc_raft.py b/harnessed_jobs/ptc_raft/v0/producer_ptc_raft.py index 6ec6300..759052c 100755 --- a/harnessed_jobs/ptc_raft/v0/producer_ptc_raft.py +++ b/harnessed_jobs/ptc_raft/v0/producer_ptc_raft.py @@ -9,6 +9,7 @@ from multiprocessor_execution import sensor_analyses def run_ptc_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) flat_files = siteUtils.dependency_glob('S*/%s_flat*flat?_*.fits' % sensor_id, jobname=siteUtils.getProcessName('flat_pair_raft_acq'), description='Flat files:') @@ -23,7 +24,7 @@ def run_ptc_task(sensor_id): results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.ptcs, - '%s_ptcs.png' % sensor_id, + '%s_ptcs.png' % file_prefix, ptc_file='%s_ptc.fits' % sensor_id) if __name__ == '__main__': diff --git a/harnessed_jobs/qa_plots_raft/v0/producer_qa_plots_raft.py b/harnessed_jobs/qa_plots_raft/v0/producer_qa_plots_raft.py index af0de62..1760cbd 100755 --- a/harnessed_jobs/qa_plots_raft/v0/producer_qa_plots_raft.py +++ b/harnessed_jobs/qa_plots_raft/v0/producer_qa_plots_raft.py @@ -4,6 +4,7 @@ """ from __future__ import print_function import os +import glob from collections import OrderedDict from lcatr.harness.helpers import dependency_glob import siteUtils @@ -44,3 +45,7 @@ def dirname_dependencyGlob(ccd_vendor, **kwds): QA_trender.processDirectory(dirname, test_type) QA_trender.plot(raft_id) + +file_prefix = '%s_%s' % (raft_id, siteUtils.getRunNumber()) +for png_file in glob.glob('*QA*.png'): + os.rename(png_file, png_file.replace(raft_id, file_prefix)) diff --git a/harnessed_jobs/qe_raft_analysis/v0/producer_qe_raft_analysis.py b/harnessed_jobs/qe_raft_analysis/v0/producer_qe_raft_analysis.py index 7d4eb88..8cf8c85 100755 --- a/harnessed_jobs/qe_raft_analysis/v0/producer_qe_raft_analysis.py +++ b/harnessed_jobs/qe_raft_analysis/v0/producer_qe_raft_analysis.py @@ -11,6 +11,7 @@ from multiprocessor_execution import sensor_analyses def run_qe_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) lambda_files = siteUtils.dependency_glob('S*/%s_lambda_flat_*.fits' % sensor_id, jobname=siteUtils.getProcessName('qe_raft_acq'), description='Lambda files:') @@ -48,7 +49,7 @@ def run_qe_task(sensor_id): plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) siteUtils.make_png_file(plots.qe, - '%s_qe.png' % sensor_id, + '%s_qe.png' % file_prefix, qe_file='%s_QE.fits' % sensor_id) try: diff --git a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py index f7d98e1..0d8b3b5 100755 --- a/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py +++ b/harnessed_jobs/read_noise_raft/v0/producer_read_noise_raft.py @@ -9,6 +9,7 @@ from multiprocessor_execution import sensor_analyses def run_read_noise_task(sensor_id): + file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) bias_files = siteUtils.dependency_glob('S*/%s_fe55_fe55_*.fits' % sensor_id, jobname=siteUtils.getProcessName('fe55_raft_acq'), description='Fe55 files for read noise:') @@ -28,7 +29,7 @@ def run_read_noise_task(sensor_id): results_file = '%s_eotest_results.fits' % sensor_id plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file) - siteUtils.make_png_file(plots.noise, '%s_noise.png' % sensor_id) + siteUtils.make_png_file(plots.noise, '%s_noise.png' % file_prefix) if __name__ == '__main__': sensor_analyses(run_read_noise_task)