From 20e0c96823503d4b42cfc0b72b557974551a5fad Mon Sep 17 00:00:00 2001 From: James Chiang Date: Mon, 8 Mar 2021 12:31:37 -0800 Subject: [PATCH 01/11] add code to plot Fe55 gains vs MJD --- python/plot_fe55_raft_gains.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 python/plot_fe55_raft_gains.py diff --git a/python/plot_fe55_raft_gains.py b/python/plot_fe55_raft_gains.py new file mode 100644 index 0000000..dc86c69 --- /dev/null +++ b/python/plot_fe55_raft_gains.py @@ -0,0 +1,48 @@ +import matplotlib.pyplot as plt +import pandas as pd +import lsst.eotest.image_utils as imutils + +__all__ = ['plot_fe55_raft_gains'] + +channel = {i: f'C{_}' for i, _ in imutils.channelIds.items()} + +def plot_fe55_raft_gains(raft_files, y_range=(1, 1.6), figsize=(12, 12)): + """ + Plot Fe55 gains for all amps for each CCD in a raft. + + Parameters + ---------- + raft_files: list + A list of pickle files containing pandas data frames with + the gains for each amp in a CCD as a function of MJD. + The filenames are assumed to be of the form + '___gain_sequence.pickle`, so that the + detector name (e.g., 'R22_S11'), raft, and run number can + be extracted. + y_range: (float, float) [(1, 1.6)] + Plotting limts of the y-axis for each plot. + figsize: (float, float) [(12, 12)] + Size of the figure for each raft. + """ + plt.figure(figsize=figsize) + for i, item in enumerate(raft_files, 1): + plt.subplot(3, 3, i) + df = pd.read_pickle(item) + mjd0 = int(min(df['mjd'])) + det_name = os.path.basename(item)[:len('R22_S11')] + amps = sorted(list(set(df['amp']))) + for amp in amps: + my_df = df.query(f'amp == {amp}') + plt.scatter(24*(my_df['mjd'] - mjd0), my_df['gain'], s=2, + label=f'{channel[amp]}') + plt.legend(fontsize='x-small', ncol=2) + plt.ylim(*y_range) + plt.title(det_name, fontsize='x-small') + plt.xlabel(f'24*(MJD - {mjd0:d})') + plt.ylabel('gain (e-/ADU)') + plt.tight_layout(rect=(0, 0, 1, 0.95)) + tokens = os.path.basename(item).split('_') + run = tokens[2] + raft = tokens[0] + plt.suptitle(f'Fe55 gain stability, {raft}, Run {run}') + plt.savefig(f'{raft}_{run}_fe55_gain_stability.png') From 01f074d55e0a7023b4b4508dcb06fb8b26de022c Mon Sep 17 00:00:00 2001 From: James Chiang Date: Mon, 8 Mar 2021 13:05:04 -0800 Subject: [PATCH 02/11] plot gain/mean(gain) instead of gain; use autoscaling of y-limits by default --- python/plot_fe55_raft_gains.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/python/plot_fe55_raft_gains.py b/python/plot_fe55_raft_gains.py index dc86c69..7e76a86 100644 --- a/python/plot_fe55_raft_gains.py +++ b/python/plot_fe55_raft_gains.py @@ -1,3 +1,5 @@ +import os +import numpy as np import matplotlib.pyplot as plt import pandas as pd import lsst.eotest.image_utils as imutils @@ -6,7 +8,8 @@ channel = {i: f'C{_}' for i, _ in imutils.channelIds.items()} -def plot_fe55_raft_gains(raft_files, y_range=(1, 1.6), figsize=(12, 12)): +def plot_fe55_raft_gains(raft_files, figsize=(12, 12), y_range=None, + outfile=None): """ Plot Fe55 gains for all amps for each CCD in a raft. @@ -19,10 +22,14 @@ def plot_fe55_raft_gains(raft_files, y_range=(1, 1.6), figsize=(12, 12)): '___gain_sequence.pickle`, so that the detector name (e.g., 'R22_S11'), raft, and run number can be extracted. - y_range: (float, float) [(1, 1.6)] - Plotting limts of the y-axis for each plot. + y_range: (float, float) [None] + Plotting limts of the y-axis for each plot. If None, then the + y-limits will be scaled by matplotlib to the data. figsize: (float, float) [(12, 12)] Size of the figure for each raft. + outfile: str [None] + Filename of output png file. If None, then a default name of + f'{raft}_{run}_fe55_gain_stability.png' will be used. """ plt.figure(figsize=figsize) for i, item in enumerate(raft_files, 1): @@ -33,16 +40,21 @@ def plot_fe55_raft_gains(raft_files, y_range=(1, 1.6), figsize=(12, 12)): amps = sorted(list(set(df['amp']))) for amp in amps: my_df = df.query(f'amp == {amp}') - plt.scatter(24*(my_df['mjd'] - mjd0), my_df['gain'], s=2, + gains = my_df['gain'].to_numpy() + frac = gains/np.mean(gains) + plt.scatter(24*(my_df['mjd'] - mjd0), frac, s=2, label=f'{channel[amp]}') plt.legend(fontsize='x-small', ncol=2) - plt.ylim(*y_range) + if y_range is not None: + plt.ylim(*y_range) plt.title(det_name, fontsize='x-small') plt.xlabel(f'24*(MJD - {mjd0:d})') - plt.ylabel('gain (e-/ADU)') + plt.ylabel('gain/mean(gain)') plt.tight_layout(rect=(0, 0, 1, 0.95)) tokens = os.path.basename(item).split('_') run = tokens[2] raft = tokens[0] plt.suptitle(f'Fe55 gain stability, {raft}, Run {run}') - plt.savefig(f'{raft}_{run}_fe55_gain_stability.png') + if outfile is None: + outfile = f'{raft}_{run}_fe55_gain_stability.png' + plt.savefig(outfile) From 1298ea23b7dad317c9500e2fbc720e9202c98a6f Mon Sep 17 00:00:00 2001 From: James Chiang Date: Mon, 8 Mar 2021 16:41:52 -0800 Subject: [PATCH 03/11] make and persist Fe55 gain stability plots --- .../v0/producer_fe55_analysis_BOT.py | 13 +++++++++++++ python/bot_eo_validators.py | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py index 54d5f5e..20f2332 100755 --- a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py +++ b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py @@ -3,9 +3,12 @@ Producer script for BOT Fe55 analysis. """ import os +import glob +from collections import defaultdict from fe55_jh_task import fe55_jh_task from gain_stability_jh_task import gain_stability_jh_task from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script +from plot_fe55_raft_gains import plot_fe55_raft_gains job_dir = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', 'fe55_analysis_BOT', 'v0') @@ -19,3 +22,13 @@ if 'gainstability' in analysis_types: gain_stability_script = os.path.join(job_dir, 'gain_stability_jh_task.py') run_python_task_or_cl_script(gain_stability_jh_task, gain_stability_script) + + # Make png files of plots of gains vs MJD for each raft. + pickle_files = sorted(glob.glob('*_gain_sequence.pickle')) + raft_files = defaultdict(list) + for item in pickle_files: + raft = os.path.basename(item)[:len('R22')] + raft_files[raft].append(item) + + for raft in raft_files: + plot_fe55_raft_gains(raft_files[raft]) diff --git a/python/bot_eo_validators.py b/python/bot_eo_validators.py index 988efd8..2c234ed 100644 --- a/python/bot_eo_validators.py +++ b/python/bot_eo_validators.py @@ -207,6 +207,11 @@ def validate_fe55(results, det_names): md = dict(DATA_PRODUCT='gain_stability_results') results.append(siteUtils.make_fileref(gain_stability_file, metadata=md)) + # Persist raft-level gain stability plots + png_files = glob.glob('*fe55_gain_stability.png') + for png_file in png_files: + md = dict(DATA_PRODUCT='gain_stability_results') + results.append(siteUtils.make_fileref(png_file, metadata=md)) report_missing_data('validate_fe55', missing_det_names) if 'gain_stability' in analysis_types: From cd4ac7e7d56845fe98ab55b44f8a78f0a883cf38 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Tue, 9 Mar 2021 11:32:45 -0800 Subject: [PATCH 04/11] add function to do full raft summary, plotting gain histories by CCD --- .../v0/producer_fe55_analysis_BOT.py | 8 ++- ...5_raft_gains.py => fe55_gain_stability.py} | 50 +++++++++++++++++-- python/flat_gain_stability.py | 2 + 3 files changed, 55 insertions(+), 5 deletions(-) rename python/{plot_fe55_raft_gains.py => fe55_gain_stability.py} (53%) diff --git a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py index 20f2332..2410d2a 100755 --- a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py +++ b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py @@ -8,7 +8,8 @@ from fe55_jh_task import fe55_jh_task from gain_stability_jh_task import gain_stability_jh_task from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script -from plot_fe55_raft_gains import plot_fe55_raft_gains +from fe55_gain_stability import plot_raft_fe55_gains_by_amp, \ + plot_all_raft_fe55_gains job_dir = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', 'fe55_analysis_BOT', 'v0') @@ -31,4 +32,7 @@ raft_files[raft].append(item) for raft in raft_files: - plot_fe55_raft_gains(raft_files[raft]) + plot_raft_fe55_gains_by_amp(raft_files[raft]) + + # Make focal plane summary plot of gain stability plotting by CCD. + plot_all_raft_fe55_gains(raft_files) diff --git a/python/plot_fe55_raft_gains.py b/python/fe55_gain_stability.py similarity index 53% rename from python/plot_fe55_raft_gains.py rename to python/fe55_gain_stability.py index 7e76a86..15e3b2f 100644 --- a/python/plot_fe55_raft_gains.py +++ b/python/fe55_gain_stability.py @@ -2,14 +2,16 @@ import numpy as np import matplotlib.pyplot as plt import pandas as pd +import siteUtils import lsst.eotest.image_utils as imutils -__all__ = ['plot_fe55_raft_gains'] +__all__ = ['plot_raft_fe55_gains_by_amp', 'plot_raft_fe55_gains_by_ccd', + 'plot_all_raft_fe55_gains'] channel = {i: f'C{_}' for i, _ in imutils.channelIds.items()} -def plot_fe55_raft_gains(raft_files, figsize=(12, 12), y_range=None, - outfile=None): +def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, + outfile=None): """ Plot Fe55 gains for all amps for each CCD in a raft. @@ -58,3 +60,45 @@ def plot_fe55_raft_gains(raft_files, figsize=(12, 12), y_range=None, if outfile is None: outfile = f'{raft}_{run}_fe55_gain_stability.png' plt.savefig(outfile) + + +def plot_raft_fe55_gains_by_ccd(raft_files, colors=None, y_range=None): + if colors is None: + colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] + raft = os.path.basename(raft_files[0])[:len('R22')] + mjd0 = None + for item, color in zip(raft_files, colors): + sensor = os.path.basename(item)[len('R22_'):len('R22_S11')] + df = pd.read_pickle(item) + if mjd0 is None: + mjd0 = int(min(df['mjd'])) + x, y = [], [] + for amp in range(1, 17): + my_df = df.query(f'amp == {amp}') + time = 24*(my_df['mjd'] - mjd0) + gain = my_df['gain']/np.mean(my_df['gain']) + x.extend(time) + y.extend(gain) + plt.scatter(x, y, s=1, color=color, label=sensor) + plt.xlabel(f'(MJD - {mjd0})*24 (hours)') + plt.ylabel('gain/mean(gain)') + plt.legend(fontsize='x-small') + plt.title(raft, fontsize='small') + if y_range is not None: + plt.ylim(*y_range) + + +def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None): + """ + Plot the flat gain stability curve for all 25 rafts in a 5x5 grid. + """ + figure = plt.figure(figsize=figsize) + rafts = sorted(list(raft_files.keys())) + for i, raft in enumerate(rafts, 1): + ax = figure.add_subplot(5, 5, i) + plot_raft_fe55_gains_by_ccd(raft_files[raft], y_range=y_range) + plt.tight_layout() + run = os.path.basename(raft_files[raft][0]).split('_')[2] + plt.suptitle(f'Run {run}') + unit_id = siteUtils.getUnitId() + plt.savefig(f'{unit_id}_{run}_fe55_gain_stability.png') diff --git a/python/flat_gain_stability.py b/python/flat_gain_stability.py index 6ce03ec..562ac0a 100644 --- a/python/flat_gain_stability.py +++ b/python/flat_gain_stability.py @@ -55,6 +55,8 @@ def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), raft_files = defaultdict(list) files = sorted(glob.glob(os.path.join(results_dir, '*flat_signal_sequence.pickle'))) + if not files: + return for item in files: raft_name = os.path.basename(item)[:len('R22')] raft_files[raft_name].append(item) From 6fb4c7137ffa85d3f1f7edcc96b8e77394a32ecf Mon Sep 17 00:00:00 2001 From: James Chiang Date: Tue, 9 Mar 2021 12:51:04 -0800 Subject: [PATCH 05/11] check if flat gain stability plot exists before trying to persist it --- python/bot_eo_validators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/bot_eo_validators.py b/python/bot_eo_validators.py index 2c234ed..5cb2165 100644 --- a/python/bot_eo_validators.py +++ b/python/bot_eo_validators.py @@ -719,9 +719,10 @@ def validate_flat_gain_stability(results, det_names): unit_id = siteUtils.getUnitId() gain_stability_plot = f'{unit_id}_{run}_flat_gain_stability.png' - md = dict(DATA_PRODUCT='flat_gain_stability_plot', LsstId=unit_id) - results.append(siteUtils.make_fileref(gain_stability_plot, - metadata=md)) + if os.path.isfile(gain_stability_plot): + md = dict(DATA_PRODUCT='flat_gain_stability_plot', LsstId=unit_id) + results.append(siteUtils.make_fileref(gain_stability_plot, + metadata=md)) return results From b095415010601c077ec7dac89304bfc7065d7d33 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Tue, 9 Mar 2021 13:45:11 -0800 Subject: [PATCH 06/11] adjust layout to make space for suptitle --- python/fe55_gain_stability.py | 2 +- python/flat_gain_stability.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/fe55_gain_stability.py b/python/fe55_gain_stability.py index 15e3b2f..1dff82d 100644 --- a/python/fe55_gain_stability.py +++ b/python/fe55_gain_stability.py @@ -97,7 +97,7 @@ def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None): for i, raft in enumerate(rafts, 1): ax = figure.add_subplot(5, 5, i) plot_raft_fe55_gains_by_ccd(raft_files[raft], y_range=y_range) - plt.tight_layout() + plt.tight_layout(rect=(0, 0, 1, 0.95)) run = os.path.basename(raft_files[raft][0]).split('_')[2] plt.suptitle(f'Run {run}') unit_id = siteUtils.getUnitId() diff --git a/python/flat_gain_stability.py b/python/flat_gain_stability.py index 562ac0a..ec5444e 100644 --- a/python/flat_gain_stability.py +++ b/python/flat_gain_stability.py @@ -67,7 +67,7 @@ def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), ax = figure.add_subplot(5, 5, i+1) plot_raft(raft_files[raft], cut=cut, divide_by_flux=divide_by_flux, y_range=y_range) - plt.tight_layout() + plt.tight_layout(rect=(0, 0, 1, 0.95)) plt.suptitle(f'Run {run}') unit_id = siteUtils.getUnitId() plt.savefig(f'{unit_id}_{run}_flat_gain_stability.png') From 5a953223aee68e181660e9525997211deb9ef948 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Tue, 9 Mar 2021 21:10:26 -0800 Subject: [PATCH 07/11] add function to plot flat gain stability for amps individually for all ccds in a raft --- python/fe55_gain_stability.py | 27 +++++++++++----- python/flat_gain_stability.py | 59 ++++++++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/python/fe55_gain_stability.py b/python/fe55_gain_stability.py index 1dff82d..b825af1 100644 --- a/python/fe55_gain_stability.py +++ b/python/fe55_gain_stability.py @@ -1,3 +1,6 @@ +""" +Module to make Fe55 gain stability plots. +""" import os import numpy as np import matplotlib.pyplot as plt @@ -11,7 +14,7 @@ channel = {i: f'C{_}' for i, _ in imutils.channelIds.items()} def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, - outfile=None): + outfile=None, acq_run=None): """ Plot Fe55 gains for all amps for each CCD in a raft. @@ -53,16 +56,22 @@ def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, plt.xlabel(f'24*(MJD - {mjd0:d})') plt.ylabel('gain/mean(gain)') plt.tight_layout(rect=(0, 0, 1, 0.95)) - tokens = os.path.basename(item).split('_') + tokens = os.path.basename(raft_files[0]).split('_') run = tokens[2] raft = tokens[0] - plt.suptitle(f'Fe55 gain stability, {raft}, Run {run}') + suptitle = f'Fe55 gain stability, {raft}, Run {run}' + if acq_run is not None: + suptitle += f' (acq. {acq_run})' + plt.suptitle(suptitle) if outfile is None: outfile = f'{raft}_{run}_fe55_gain_stability.png' plt.savefig(outfile) def plot_raft_fe55_gains_by_ccd(raft_files, colors=None, y_range=None): + """ + Plot Fe55 gains for each ccd in a raft, aggregating over amps. + """ if colors is None: colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] raft = os.path.basename(raft_files[0])[:len('R22')] @@ -88,17 +97,21 @@ def plot_raft_fe55_gains_by_ccd(raft_files, colors=None, y_range=None): plt.ylim(*y_range) -def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None): +def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None, + acq_run=None): """ Plot the flat gain stability curve for all 25 rafts in a 5x5 grid. """ figure = plt.figure(figsize=figsize) rafts = sorted(list(raft_files.keys())) for i, raft in enumerate(rafts, 1): - ax = figure.add_subplot(5, 5, i) + figure.add_subplot(5, 5, i) plot_raft_fe55_gains_by_ccd(raft_files[raft], y_range=y_range) plt.tight_layout(rect=(0, 0, 1, 0.95)) - run = os.path.basename(raft_files[raft][0]).split('_')[2] - plt.suptitle(f'Run {run}') + run = os.path.basename(raft_files[rafts[0]][0]).split('_')[2] + suptitle = f'Fe55 gain stability, Run {run}' + if acq_run is not None: + suptitle += f' (acq. {acq_run})' + plt.suptitle(suptitle) unit_id = siteUtils.getUnitId() plt.savefig(f'{unit_id}_{run}_fe55_gain_stability.png') diff --git a/python/flat_gain_stability.py b/python/flat_gain_stability.py index ec5444e..e7f8327 100644 --- a/python/flat_gain_stability.py +++ b/python/flat_gain_stability.py @@ -7,14 +7,62 @@ import numpy as np import matplotlib.pyplot as plt import pandas as pd +import lsst.eotest.image_utils as imutils import siteUtils -__all__ = ['plot_raft', 'plot_all_rafts'] +__all__ = ['plot_raft_by_amp', 'plot_raft', 'plot_all_rafts'] + + +channel = {i: f'C{_}' for i, _ in imutils.channelIds.items()} + + +def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, + figsize=(12, 12), y_range=None, outfile=None, + acq_run=None): + """ + Plot flat gain stability for each amp individually. + """ + plt.figure(figsize=figsize) + for i, item in enumerate(raft_files, i): + plt.subplot(3, 3, i) + df = pd.read_pickle(item) + if cut: + df = df.query(cut) + flux = df['flux']/np.mean(df['flux']) + mjd0 = int(min(df['mjd'])) + for amp in range(1, 17): + try: + signal = df[f'amp{amp:02d}'] + except KeyError: + continue + if divide_by_flux: + signal /= flux + signal /= np.mean(signal) + plt.scatter(24*(df['mjd'] - mjd0), signal, s=2, label=channel[amp]) + plt.xlabel(f'(MJD - {mjd0})*24 (hours)') + plt.ylabel('counts/mean(counts)') + if y_range is not None: + plt.ylim(*y_range) + plt.legend(fontsize='x-small') + plt.tight_layout(rect=(0, 0, 1, 0.95)) + tokens = os.path.basename(raft_files[0]).split('_') + run = tokens[2] + raft = tokens[0] + suptitle = f'flat gain stability, {raft}, Run {run}' + if acq_run is not None: + suptitle += f' (acq. {acq_run})' + plt.suptitle(suptitle, fontsize='x-small') + if outfile is None: + outfile = f'{raft}_{run}_flat_gain_stability.png' + plt.savefig(outfile) def plot_raft(raft_files, cut=None, divide_by_flux=True, y_range=None, colors=None): + """ + Plot flat gain stability for all ccds in a raft, aggregating over amps. + """ if colors is None: colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] raft = os.path.basename(raft_files[0])[:len('R22')] @@ -48,7 +96,7 @@ def plot_raft(raft_files, cut=None, divide_by_flux=True, y_range=None, def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), - divide_by_flux=True, figsize=(18, 18)): + divide_by_flux=True, figsize=(18, 18), acq_run=None): """ Plot the flat gain stability curve for all 25 rafts in a 5x5 grid. """ @@ -64,10 +112,13 @@ def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), figure = plt.figure(figsize=figsize) rafts = sorted(list(raft_files.keys())) for i, raft in enumerate(rafts): - ax = figure.add_subplot(5, 5, i+1) + figure.add_subplot(5, 5, i+1) plot_raft(raft_files[raft], cut=cut, divide_by_flux=divide_by_flux, y_range=y_range) plt.tight_layout(rect=(0, 0, 1, 0.95)) - plt.suptitle(f'Run {run}') + suptitle = f'flat gain stability, Run {run}' + if acq_run is not None: + suptitle += f' (acq. {acq_run})' + plt.suptitle(suptitle) unit_id = siteUtils.getUnitId() plt.savefig(f'{unit_id}_{run}_flat_gain_stability.png') From b9b78a02301fb40a49962f5d59d80c8be0b1cb75 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Wed, 10 Mar 2021 12:14:05 -0800 Subject: [PATCH 08/11] make raft-level plots of flat gain stability, rendering amps individually; set acq_run option in plotting function calls --- .../v0/producer_fe55_analysis_BOT.py | 7 ++++--- .../v0/producer_flat_gain_stability_BOT.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py index 2410d2a..f010f63 100755 --- a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py +++ b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py @@ -31,8 +31,9 @@ raft = os.path.basename(item)[:len('R22')] raft_files[raft].append(item) + acq_run = os.environ.get('LCATR_ACQ_RUN', None) for raft in raft_files: - plot_raft_fe55_gains_by_amp(raft_files[raft]) + plot_raft_fe55_gains_by_amp(raft_files[raft], acq_run=acq_run) - # Make focal plane summary plot of gain stability plotting by CCD. - plot_all_raft_fe55_gains(raft_files) + # Make focal plane summary plot of gain stability, aggregating by CCD. + plot_all_raft_fe55_gains(raft_files, acq_run=acq_run) diff --git a/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py b/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py index 04718a5..d8af353 100755 --- a/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py +++ b/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py @@ -3,9 +3,11 @@ Producer script for BOT flat gain stability analysis. """ import os +import glob +from collections import defaultdict import siteUtils from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script -from flat_gain_stability import plot_all_rafts +from flat_gain_stability import plot_all_rafts, plot_raft_by_amp from flat_gain_stability_jh_task import flat_gain_stability_jh_task if 'gainstability' in get_analysis_types(): @@ -16,4 +18,16 @@ run_python_task_or_cl_script(flat_gain_stability_jh_task, flat_gain_stability_script) - plot_all_rafts(siteUtils.getRunNumber(), y_range=None) + # Make png files of plots of gains vs MJD for each raft. + pickle_files = sorted(glob.glob('*flat_signal_sequence.pickle')) + raft_files = defaultdict(list) + for item in pickle_files: + raft = os.path.basename(item)[:len('R22')] + raft_files[raft].append(item) + + acq_run = os.environ.get('LCATR_ACQ_RUN', None) + for raft in raft_files: + plot_raft_by_amp(raft_files[raft], acq_run=acq_run) + + # Make focal plane summary plot of gain stability, aggregating by CCD + plot_all_rafts(siteUtils.getRunNumber(), acq_run=acq_run) From a938e89d6b9492358d6fee89eb3fb48738f9ffb1 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Wed, 10 Mar 2021 19:33:00 -0800 Subject: [PATCH 09/11] persist flat gain stability plots for individual rafts; standardize axis labels and legend formats --- python/bot_eo_validators.py | 11 ++++++----- python/fe55_gain_stability.py | 2 +- python/flat_gain_stability.py | 16 +++++++++------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/python/bot_eo_validators.py b/python/bot_eo_validators.py index 5cb2165..002c929 100644 --- a/python/bot_eo_validators.py +++ b/python/bot_eo_validators.py @@ -718,11 +718,12 @@ def validate_flat_gain_stability(results, det_names): report_missing_data('validate_flat_gain_stability', missing_det_names) unit_id = siteUtils.getUnitId() - gain_stability_plot = f'{unit_id}_{run}_flat_gain_stability.png' - if os.path.isfile(gain_stability_plot): - md = dict(DATA_PRODUCT='flat_gain_stability_plot', LsstId=unit_id) - results.append(siteUtils.make_fileref(gain_stability_plot, - metadata=md)) + png_files = glob.glob('*flat_gain_stability.png') + for png_file in png_files: + md = dict(DATA_PRODUCT='flat_gain_stability_plot') + if unit_id in png_file: + md['LsstId'] = unit_id + results.append(siteUtils.make_fileref(png_file, metadata=md)) return results diff --git a/python/fe55_gain_stability.py b/python/fe55_gain_stability.py index b825af1..29686cf 100644 --- a/python/fe55_gain_stability.py +++ b/python/fe55_gain_stability.py @@ -53,7 +53,7 @@ def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, if y_range is not None: plt.ylim(*y_range) plt.title(det_name, fontsize='x-small') - plt.xlabel(f'24*(MJD - {mjd0:d})') + plt.xlabel(f'(MJD - {mjd0:d})*24 (hours)') plt.ylabel('gain/mean(gain)') plt.tight_layout(rect=(0, 0, 1, 0.95)) tokens = os.path.basename(raft_files[0]).split('_') diff --git a/python/flat_gain_stability.py b/python/flat_gain_stability.py index e7f8327..c17de28 100644 --- a/python/flat_gain_stability.py +++ b/python/flat_gain_stability.py @@ -24,13 +24,14 @@ def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, Plot flat gain stability for each amp individually. """ plt.figure(figsize=figsize) - for i, item in enumerate(raft_files, i): + for i, item in enumerate(raft_files, 1): plt.subplot(3, 3, i) df = pd.read_pickle(item) if cut: df = df.query(cut) flux = df['flux']/np.mean(df['flux']) mjd0 = int(min(df['mjd'])) + det_name = os.path.basename(item)[:len('R22_S11')] for amp in range(1, 17): try: signal = df[f'amp{amp:02d}'] @@ -40,11 +41,12 @@ def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, signal /= flux signal /= np.mean(signal) plt.scatter(24*(df['mjd'] - mjd0), signal, s=2, label=channel[amp]) - plt.xlabel(f'(MJD - {mjd0})*24 (hours)') - plt.ylabel('counts/mean(counts)') + plt.legend(fontsize='x-small', ncol=2) if y_range is not None: plt.ylim(*y_range) - plt.legend(fontsize='x-small') + plt.title(det_name, fontsize='x-small') + plt.xlabel(f'(MJD - {mjd0})*24 (hours)') + plt.ylabel('counts/mean(counts)') plt.tight_layout(rect=(0, 0, 1, 0.95)) tokens = os.path.basename(raft_files[0]).split('_') run = tokens[2] @@ -52,7 +54,7 @@ def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, suptitle = f'flat gain stability, {raft}, Run {run}' if acq_run is not None: suptitle += f' (acq. {acq_run})' - plt.suptitle(suptitle, fontsize='x-small') + plt.suptitle(suptitle) if outfile is None: outfile = f'{raft}_{run}_flat_gain_stability.png' plt.savefig(outfile) @@ -89,7 +91,7 @@ def plot_raft(raft_files, cut=None, divide_by_flux=True, y_range=None, plt.scatter(x, y, s=1, color=color, label=sensor) plt.xlabel(f'(MJD - {mjd_min})*24 (hours)') plt.ylabel('counts/mean(counts)') - plt.legend(fontsize='x-small') + plt.legend(fontsize='x-small', ncol=2) plt.title(raft, fontsize='small') if y_range is not None: plt.ylim(*y_range) @@ -116,7 +118,7 @@ def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), plot_raft(raft_files[raft], cut=cut, divide_by_flux=divide_by_flux, y_range=y_range) plt.tight_layout(rect=(0, 0, 1, 0.95)) - suptitle = f'flat gain stability, Run {run}' + suptitle = f'Flat gain stability, Run {run}' if acq_run is not None: suptitle += f' (acq. {acq_run})' plt.suptitle(suptitle) From a19b9705a69f9307248cd5a97ecd5dbd3e0ab5f6 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Thu, 11 Mar 2021 16:24:48 -0800 Subject: [PATCH 10/11] add RaftCcdOrder class to standardize plotting order of CCDs; use append_acq_run in gain stability producer scripts --- .../v0/producer_fe55_analysis_BOT.py | 9 +++--- .../v0/producer_flat_gain_stability_BOT.py | 5 ++-- python/bot_eo_analyses.py | 29 +++++++++++++++++++ python/fe55_gain_stability.py | 26 ++++++++--------- python/flat_gain_stability.py | 21 ++++++-------- 5 files changed, 57 insertions(+), 33 deletions(-) diff --git a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py index f010f63..cc98dee 100755 --- a/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py +++ b/harnessed_jobs/fe55_analysis_BOT/v0/producer_fe55_analysis_BOT.py @@ -5,11 +5,11 @@ import os import glob from collections import defaultdict -from fe55_jh_task import fe55_jh_task -from gain_stability_jh_task import gain_stability_jh_task from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script from fe55_gain_stability import plot_raft_fe55_gains_by_amp, \ plot_all_raft_fe55_gains +from fe55_jh_task import fe55_jh_task +from gain_stability_jh_task import gain_stability_jh_task job_dir = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', 'fe55_analysis_BOT', 'v0') @@ -31,9 +31,8 @@ raft = os.path.basename(item)[:len('R22')] raft_files[raft].append(item) - acq_run = os.environ.get('LCATR_ACQ_RUN', None) for raft in raft_files: - plot_raft_fe55_gains_by_amp(raft_files[raft], acq_run=acq_run) + plot_raft_fe55_gains_by_amp(raft_files[raft]) # Make focal plane summary plot of gain stability, aggregating by CCD. - plot_all_raft_fe55_gains(raft_files, acq_run=acq_run) + plot_all_raft_fe55_gains(raft_files) diff --git a/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py b/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py index d8af353..a313d71 100755 --- a/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py +++ b/harnessed_jobs/flat_gain_stability_BOT/v0/producer_flat_gain_stability_BOT.py @@ -25,9 +25,8 @@ raft = os.path.basename(item)[:len('R22')] raft_files[raft].append(item) - acq_run = os.environ.get('LCATR_ACQ_RUN', None) for raft in raft_files: - plot_raft_by_amp(raft_files[raft], acq_run=acq_run) + plot_raft_by_amp(raft_files[raft]) # Make focal plane summary plot of gain stability, aggregating by CCD - plot_all_rafts(siteUtils.getRunNumber(), acq_run=acq_run) + plot_all_rafts(siteUtils.getRunNumber()) diff --git a/python/bot_eo_analyses.py b/python/bot_eo_analyses.py index f7a8967..23e8e42 100644 --- a/python/bot_eo_analyses.py +++ b/python/bot_eo_analyses.py @@ -37,6 +37,7 @@ 'append_acq_run', 'make_title', 'glob_pattern', + 'raft_ccd_order', 'get_mask_files', 'get_amplifier_gains', 'medianed_dark_frame', @@ -182,6 +183,34 @@ def get_mask_files(det_name): return mask_files +class RaftCcdOrder: + """ + Class to sort eotest output filenames by order of appearance of CCDs + in a raft for the standard DM-based focal plane orientation, for + science rafts and corner rafts (specifically R44), respectively: + S20 S21 S22 SG0 + S10 S11 S12 SW1 + S00 S01 S02 SW0 SG1 + """ + ccd_order = 'S20 S21 S22 S10 S11 S12 S00 S01 S02 SG0 SW1 SW0 SG1'.split() + def _ccd_key(self, filename): + """ + Key function to use in sorted(...). Filename should be of the + form '{raft}_{sensor}_{run}_...'. + """ + ccd_slot = os.path.basename(filename).split('_')[1] + return self.ccd_order.index(ccd_slot) + + def sorted(self, file_list): + """ + Return a sorted file list based on CCD slot name. + """ + return sorted(file_list, key=self._ccd_key) + + +raft_ccd_order = RaftCcdOrder() + + def make_file_prefix(run, component_name): """ Compose the run number and component name into string prefix diff --git a/python/fe55_gain_stability.py b/python/fe55_gain_stability.py index 29686cf..f2424fc 100644 --- a/python/fe55_gain_stability.py +++ b/python/fe55_gain_stability.py @@ -7,14 +7,17 @@ import pandas as pd import siteUtils import lsst.eotest.image_utils as imutils +from bot_eo_analyses import raft_ccd_order, append_acq_run __all__ = ['plot_raft_fe55_gains_by_amp', 'plot_raft_fe55_gains_by_ccd', 'plot_all_raft_fe55_gains'] + channel = {i: f'C{_}' for i, _ in imutils.channelIds.items()} + def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, - outfile=None, acq_run=None): + outfile=None): """ Plot Fe55 gains for all amps for each CCD in a raft. @@ -27,17 +30,18 @@ def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, '___gain_sequence.pickle`, so that the detector name (e.g., 'R22_S11'), raft, and run number can be extracted. + figsize: (float, float) [(12, 12)] + Size of the figure for each raft. y_range: (float, float) [None] Plotting limts of the y-axis for each plot. If None, then the y-limits will be scaled by matplotlib to the data. - figsize: (float, float) [(12, 12)] - Size of the figure for each raft. outfile: str [None] Filename of output png file. If None, then a default name of f'{raft}_{run}_fe55_gain_stability.png' will be used. """ + sorted_raft_files = raft_ccd_order.sorted(raft_files) plt.figure(figsize=figsize) - for i, item in enumerate(raft_files, 1): + for i, item in enumerate(sorted_raft_files, 1): plt.subplot(3, 3, i) df = pd.read_pickle(item) mjd0 = int(min(df['mjd'])) @@ -59,9 +63,7 @@ def plot_raft_fe55_gains_by_amp(raft_files, figsize=(12, 12), y_range=None, tokens = os.path.basename(raft_files[0]).split('_') run = tokens[2] raft = tokens[0] - suptitle = f'Fe55 gain stability, {raft}, Run {run}' - if acq_run is not None: - suptitle += f' (acq. {acq_run})' + suptitle = append_acq_run(f'Fe55 gain stability, {raft}, Run {run}') plt.suptitle(suptitle) if outfile is None: outfile = f'{raft}_{run}_fe55_gain_stability.png' @@ -76,7 +78,8 @@ def plot_raft_fe55_gains_by_ccd(raft_files, colors=None, y_range=None): colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] raft = os.path.basename(raft_files[0])[:len('R22')] mjd0 = None - for item, color in zip(raft_files, colors): + sorted_raft_files = raft_ccd_order.sorted(raft_files) + for item, color in zip(sorted_raft_files, colors): sensor = os.path.basename(item)[len('R22_'):len('R22_S11')] df = pd.read_pickle(item) if mjd0 is None: @@ -97,8 +100,7 @@ def plot_raft_fe55_gains_by_ccd(raft_files, colors=None, y_range=None): plt.ylim(*y_range) -def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None, - acq_run=None): +def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None): """ Plot the flat gain stability curve for all 25 rafts in a 5x5 grid. """ @@ -109,9 +111,7 @@ def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None, plot_raft_fe55_gains_by_ccd(raft_files[raft], y_range=y_range) plt.tight_layout(rect=(0, 0, 1, 0.95)) run = os.path.basename(raft_files[rafts[0]][0]).split('_')[2] - suptitle = f'Fe55 gain stability, Run {run}' - if acq_run is not None: - suptitle += f' (acq. {acq_run})' + suptitle = append_acq_run(f'Fe55 gain stability, Run {run}') plt.suptitle(suptitle) unit_id = siteUtils.getUnitId() plt.savefig(f'{unit_id}_{run}_fe55_gain_stability.png') diff --git a/python/flat_gain_stability.py b/python/flat_gain_stability.py index c17de28..a64d12e 100644 --- a/python/flat_gain_stability.py +++ b/python/flat_gain_stability.py @@ -9,7 +9,7 @@ import pandas as pd import lsst.eotest.image_utils as imutils import siteUtils - +from bot_eo_analyses import raft_ccd_order, append_acq_run __all__ = ['plot_raft_by_amp', 'plot_raft', 'plot_all_rafts'] @@ -18,13 +18,13 @@ def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, - figsize=(12, 12), y_range=None, outfile=None, - acq_run=None): + figsize=(12, 12), y_range=None, outfile=None): """ Plot flat gain stability for each amp individually. """ + sorted_raft_files = raft_ccd_order.sorted(raft_files) plt.figure(figsize=figsize) - for i, item in enumerate(raft_files, 1): + for i, item in enumerate(sorted_raft_files, 1): plt.subplot(3, 3, i) df = pd.read_pickle(item) if cut: @@ -51,9 +51,7 @@ def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, tokens = os.path.basename(raft_files[0]).split('_') run = tokens[2] raft = tokens[0] - suptitle = f'flat gain stability, {raft}, Run {run}' - if acq_run is not None: - suptitle += f' (acq. {acq_run})' + suptitle = append_acq_run(f'flat gain stability, {raft}, Run {run}') plt.suptitle(suptitle) if outfile is None: outfile = f'{raft}_{run}_flat_gain_stability.png' @@ -69,7 +67,8 @@ def plot_raft(raft_files, cut=None, divide_by_flux=True, y_range=None, colors = plt.rcParams['axes.prop_cycle'].by_key()['color'] raft = os.path.basename(raft_files[0])[:len('R22')] mjd_min = None - for item, color in zip(raft_files, colors): + sorted_raft_files = raft_ccd_order.sorted(raft_files) + for item, color in zip(sorted_raft_files, colors): sensor = os.path.basename(item)[len('R22_'):len('R22_S11')] raw_df = pd.read_pickle(item) df = raw_df.query(cut) if cut is not None else raw_df @@ -98,7 +97,7 @@ def plot_raft(raft_files, cut=None, divide_by_flux=True, y_range=None, def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), - divide_by_flux=True, figsize=(18, 18), acq_run=None): + divide_by_flux=True, figsize=(18, 18)): """ Plot the flat gain stability curve for all 25 rafts in a 5x5 grid. """ @@ -118,9 +117,7 @@ def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002), plot_raft(raft_files[raft], cut=cut, divide_by_flux=divide_by_flux, y_range=y_range) plt.tight_layout(rect=(0, 0, 1, 0.95)) - suptitle = f'Flat gain stability, Run {run}' - if acq_run is not None: - suptitle += f' (acq. {acq_run})' + suptitle = append_acq_run(f'Flat gain stability, Run {run}') plt.suptitle(suptitle) unit_id = siteUtils.getUnitId() plt.savefig(f'{unit_id}_{run}_flat_gain_stability.png') From 34ec77b1d66041abd11260b3d9b60876eb5b04f5 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Thu, 11 Mar 2021 19:16:10 -0800 Subject: [PATCH 11/11] tweak legend for Fe55 plots and title for flat gain stability plot --- python/fe55_gain_stability.py | 2 +- python/flat_gain_stability.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/fe55_gain_stability.py b/python/fe55_gain_stability.py index f2424fc..13e6b73 100644 --- a/python/fe55_gain_stability.py +++ b/python/fe55_gain_stability.py @@ -94,7 +94,7 @@ def plot_raft_fe55_gains_by_ccd(raft_files, colors=None, y_range=None): plt.scatter(x, y, s=1, color=color, label=sensor) plt.xlabel(f'(MJD - {mjd0})*24 (hours)') plt.ylabel('gain/mean(gain)') - plt.legend(fontsize='x-small') + plt.legend(fontsize='x-small', ncol=2) plt.title(raft, fontsize='small') if y_range is not None: plt.ylim(*y_range) diff --git a/python/flat_gain_stability.py b/python/flat_gain_stability.py index a64d12e..70d9d80 100644 --- a/python/flat_gain_stability.py +++ b/python/flat_gain_stability.py @@ -51,7 +51,7 @@ def plot_raft_by_amp(raft_files, cut=None, divide_by_flux=True, tokens = os.path.basename(raft_files[0]).split('_') run = tokens[2] raft = tokens[0] - suptitle = append_acq_run(f'flat gain stability, {raft}, Run {run}') + suptitle = append_acq_run(f'Flat gain stability, {raft}, Run {run}') plt.suptitle(suptitle) if outfile is None: outfile = f'{raft}_{run}_flat_gain_stability.png'