From 743d89b24c893116835eb34b056269bf00067e03 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 27 Sep 2019 16:02:11 -0700 Subject: [PATCH 1/5] add divisidero tearing analysis to raft-level tearing detection job --- .../v0/producer_tearing_detection.py | 29 +++++++++++++++ .../v0/validator_tearing_detection.py | 37 ++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py b/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py index 3530aca..1af4334 100755 --- a/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py +++ b/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py @@ -42,8 +42,37 @@ def run_tearing_detection(sensor_id): with open('%s_tearing_stats.pkl' % file_prefix, 'wb') as output: pickle.dump(tearing_stats, output) + if __name__ == '__main__': + import os + import json + import matplotlib.pyplot as plt + import siteUtils + import camera_components + import lsst.eotest.sensor as sensorTest from multiprocessor_execution import sensor_analyses processes = 9 # Reserve 1 process per CCD. sensor_analyses(run_tearing_detection, processes=processes) + + # Divisidero tearing analysis. + run = siteUtils.getRunNumber() + raft_unit_id = siteUtils.getUnitId() + files = siteUtils.dependency_glob('*median_sflat.fits', + jobname='dark_defects_raft', + description='Superflat files:') + raft = camera_components.Raft.create_from_etrav(raft_unit_id, + db_name='Prod') + det_map = dict([(sensor_id, slot) for slot, sensor_id in raft.items()]) + + sflat_files = dict() + for item in files: + slot = det_map[os.path.basename(item).split('_')[0]] + sflat_files[slot] = [item] + + max_divisidero_tearing \ + = sensorTest.ana_divisidero_tearing(sflat_files, raft_unit_id, run) + plt.savefig('f{raft_unit_id}_{run}_divisidero.png') + + with open(f'{raft_unit_id}_{run}_max_divisidero.json', 'w') as fd: + json.dump(max_divisidero_tearing, fd) diff --git a/harnessed_jobs/tearing_detection/v0/validator_tearing_detection.py b/harnessed_jobs/tearing_detection/v0/validator_tearing_detection.py index a3aa29a..51aaf60 100755 --- a/harnessed_jobs/tearing_detection/v0/validator_tearing_detection.py +++ b/harnessed_jobs/tearing_detection/v0/validator_tearing_detection.py @@ -3,18 +3,21 @@ Validator script for raft-level flat pairs analysis. """ import glob +import json import pickle import lcatr.schema import siteUtils import camera_components from tearing_detection import persist_tearing_png_files -raft_id = siteUtils.getUnitId() -raft = camera_components.Raft.create_from_etrav(raft_id) +raft_unit_id = siteUtils.getUnitId() +raft = camera_components.Raft.create_from_etrav(raft_unit_id) +det_map = dict() results = [] schema = lcatr.schema.get('tearing_detection') for slot, sensor_id in raft.items(): + det_map[slot] = sensor_id file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber()) with open('%s_tearing_stats.pkl' % file_prefix, 'rb') as input_: tearing_stats = pickle.load(input_) @@ -27,6 +30,36 @@ png_files = sorted(glob.glob('*_tearing.png')) results.extend(persist_tearing_png_files(png_files)) +divisidero_plot = glob.glob('*_divisidero.png')[0] +md = dict(DATA_PRODUCT='divisidero_tearing_plot', + LsstId=raft_unit_id) +results.append(siteUtils.make_fileref(divisidero_plot, metadata=md)) + +with open(glob.glob('*max_divisidero.json')[0], 'r') as fd: + max_devs = json.load(fd) + +fields = ('max_deviation_10_11', + 'max_deviation_11_12', + 'max_deviation_12_13', + 'max_deviation_13_14', + 'max_deviation_14_15', + 'max_deviation_15_16', + 'max_deviation_16_17', + 'max_deviation_00_01', + 'max_deviation_01_02', + 'max_deviation_02_03', + 'max_deviation_03_04', + 'max_deviation_04_05', + 'max_deviation_05_06', + 'max_deviation_06_07') + +divisidero_schema = lcatr.schema.get('divisidero_tearing') +for slot, values in max_devs.items(): + data = {field: max_dev for field, max_dev in zip(fields, values)} + results.append(lcatr.schema.valid(divisidero_schema, slot=slot, + sensor_id=det_map[slot], + **data)) + results.extend(siteUtils.jobInfo()) lcatr.schema.write_file(results) lcatr.schema.validate_file() From b5cd28f68df085df99572f74bdc2b39828f47793 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 28 Sep 2019 12:17:46 -0700 Subject: [PATCH 2/5] bug-fix for formatting of divisidero png file --- .../tearing_detection/v0/producer_tearing_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py b/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py index 1af4334..c5ea9d0 100755 --- a/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py +++ b/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py @@ -72,7 +72,7 @@ def run_tearing_detection(sensor_id): max_divisidero_tearing \ = sensorTest.ana_divisidero_tearing(sflat_files, raft_unit_id, run) - plt.savefig('f{raft_unit_id}_{run}_divisidero.png') + plt.savefig(f'{raft_unit_id}_{run}_divisidero.png') with open(f'{raft_unit_id}_{run}_max_divisidero.json', 'w') as fd: json.dump(max_divisidero_tearing, fd) From 8a5bff2e752d578dfb04029525f23fe15b5e72b0 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 28 Sep 2019 14:55:08 -0700 Subject: [PATCH 3/5] add schema file for max deviations for divisidero tearing --- schemas/divisidero_tearing.schema | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 schemas/divisidero_tearing.schema diff --git a/schemas/divisidero_tearing.schema b/schemas/divisidero_tearing.schema new file mode 100644 index 0000000..8f291cf --- /dev/null +++ b/schemas/divisidero_tearing.schema @@ -0,0 +1,21 @@ +# -*- python -*- +{ + 'schema_name' : 'divisidero_tearing', + 'schema_version' : 0, + 'slot' : str, + 'sensor_id' : str, + 'max_deviation_10_11' : float, + 'max_deviation_11_12' : float, + 'max_deviation_12_13' : float, + 'max_deviation_13_14' : float, + 'max_deviation_14_15' : float, + 'max_deviation_15_16' : float, + 'max_deviation_16_17' : float, + 'max_deviation_00_01' : float, + 'max_deviation_01_02' : float, + 'max_deviation_02_03' : float, + 'max_deviation_03_04' : float, + 'max_deviation_04_05' : float, + 'max_deviation_05_06' : float, + 'max_deviation_06_07' : float +} From db25a127403b89495344732fe48917437f52bb65 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 28 Sep 2019 18:28:10 -0700 Subject: [PATCH 4/5] add divisidero tearing analysis to tearing_BOT job --- harnessed_jobs/tearing_BOT/v0/producer_tearing_BOT.py | 10 ++++++++++ .../tearing_detection/v0/producer_tearing_detection.py | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/harnessed_jobs/tearing_BOT/v0/producer_tearing_BOT.py b/harnessed_jobs/tearing_BOT/v0/producer_tearing_BOT.py index ea68044..c64c73b 100755 --- a/harnessed_jobs/tearing_BOT/v0/producer_tearing_BOT.py +++ b/harnessed_jobs/tearing_BOT/v0/producer_tearing_BOT.py @@ -3,7 +3,9 @@ Producer script for BOT tearing analysis. """ import os +from camera_components import camera_info from tearing_jh_task import tearing_jh_task +from raft_divisidero_tearing import raft_divisidero_tearing from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script if 'tearing' in get_analysis_types(): @@ -11,3 +13,11 @@ = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', 'tearing_BOT', 'v0', 'tearing_jh_task.py') run_python_task_or_cl_script(tearing_jh_task, tearing_jh_task_script) + + divisidero_jh_task_script \ + = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', + 'tearing_BOT', 'v0', 'raft_divisidero_tearing.py') + installed_rafts = camera_info.get_installed_raft_names() + run_python_task_or_cl_script(raft_divisidero_tearing, + divisidero_jh_task_script, + device_names=installed_rafts) diff --git a/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py b/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py index c5ea9d0..835e26b 100755 --- a/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py +++ b/harnessed_jobs/tearing_detection/v0/producer_tearing_detection.py @@ -49,7 +49,7 @@ def run_tearing_detection(sensor_id): import matplotlib.pyplot as plt import siteUtils import camera_components - import lsst.eotest.sensor as sensorTest + import lsst.eotest.raft as raftTest from multiprocessor_execution import sensor_analyses processes = 9 # Reserve 1 process per CCD. @@ -68,10 +68,10 @@ def run_tearing_detection(sensor_id): sflat_files = dict() for item in files: slot = det_map[os.path.basename(item).split('_')[0]] - sflat_files[slot] = [item] + sflat_files[slot] = item max_divisidero_tearing \ - = sensorTest.ana_divisidero_tearing(sflat_files, raft_unit_id, run) + = raftTest.ana_divisidero_tearing(sflat_files, raft_unit_id, run) plt.savefig(f'{raft_unit_id}_{run}_divisidero.png') with open(f'{raft_unit_id}_{run}_max_divisidero.json', 'w') as fd: From 33160874bcbf5e24f40f56e04b25ba1c97bdf857 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sat, 28 Sep 2019 18:28:46 -0700 Subject: [PATCH 5/5] add command line script for running divisider tearing analysis on remote nodes --- .../tearing_BOT/v0/raft_divisidero_tearing.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 harnessed_jobs/tearing_BOT/v0/raft_divisidero_tearing.py diff --git a/harnessed_jobs/tearing_BOT/v0/raft_divisidero_tearing.py b/harnessed_jobs/tearing_BOT/v0/raft_divisidero_tearing.py new file mode 100755 index 0000000..92cac8c --- /dev/null +++ b/harnessed_jobs/tearing_BOT/v0/raft_divisidero_tearing.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +""" +Command-line script for divisidero tearing analysis of BOT data. +""" +def raft_divisidero_tearing(raft_name): + """JH version of divisidero tearing analysis of BOT data.""" + import json + import matplotlib.pyplot as plt + import siteUtils + import lsst.eotest.raft as raftTest + from bot_eo_analyses import get_raft_files_by_slot + + run = siteUtils.getRunNumber() + + sflat_files = get_raft_files_by_slot(raft_name, 'median_sflat.fits') + + max_divisidero_tearing \ + = raftTest.ana_divisidero_tearing(sflat_files, raft_name, run) + plt.savefig(f'{raft_name}_{run}_divisidero.png') + + with open(f'{raft_name}_{run}_max_divisidero.json', 'w') as fd: + json.dump(max_divisidero_tearing, fd) + +if __name__ == '__main__': + import sys + raft_name = sys.argv[1] + raft_divisidero_tearing(raft_name)