From d1cd12c53ee5ab2a216e0790d1bb59488f9a06ce Mon Sep 17 00:00:00 2001 From: Bryson Cale Date: Thu, 4 Jul 2024 12:54:25 -0700 Subject: [PATCH 1/3] Minor refactoring of parse_subarray_map_step and test, not completed --- .../parse_subarray_map_step.py | 37 ++++++++----- .../tests/test_parse_subarray_map.py | 55 +++++++++++++++++++ 2 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 liger_iris_pipeline/tests/test_parse_subarray_map.py diff --git a/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py b/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py index 62e05ad..db324c6 100644 --- a/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py +++ b/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py @@ -1,7 +1,8 @@ import numpy as np from jwst.stpipe import Step -from jwst import datamodels +from .. import datamodels +import stdatamodels __all__ = ["ParseSubarrayMapStep"] @@ -34,18 +35,26 @@ class ParseSubarrayMapStep(Step): def process(self, input): - with datamodels.open(input) as input_model: - - if "subarr_map" in input_model: - self.log.info("Parsing the SUBARR_MAP extension") - result = input_model.copy() - for each in parse_subarray_map(result["subarr_map"]): - result.meta.subarray_map.append(each) - result.dq[result["subarr_map"] != 0] = np.bitwise_or( - result.dq[result["subarr_map"] != 0], 2 ** SUBARRAY_DQ_BIT - ) - else: - self.log.info("No SUBARR_MAP extension found") - result = input_model + if isinstance(input, str): + input_model = datamodels.open(input) + else: + input_model = input + + if "subarr_map" in input_model: + + self.log.info("Parsing the SUBARR_MAP extension") + + result = input_model.copy() + + for each in parse_subarray_map(result["subarr_map"]): + result.meta.subarray_map.append(each) + + result.dq[result["subarr_map"] != 0] = np.bitwise_or( + result.dq[result["subarr_map"] != 0], + 2 ** SUBARRAY_DQ_BIT + ) + else: + self.log.info("No SUBARR_MAP extension found") + result = input_model return result diff --git a/liger_iris_pipeline/tests/test_parse_subarray_map.py b/liger_iris_pipeline/tests/test_parse_subarray_map.py new file mode 100644 index 0000000..57df543 --- /dev/null +++ b/liger_iris_pipeline/tests/test_parse_subarray_map.py @@ -0,0 +1,55 @@ +# Imports +import liger_iris_pipeline +liger_iris_pipeline.monkeypatch_jwst_datamodels() +from liger_iris_pipeline.parse_subarray_map.parse_subarray_map_step import parse_subarray_map +from liger_iris_pipeline import ParseSubarrayMapStep +import numpy as np + +def set_subarray_mask(mask_array, subarray_index, xstart, ystart, xsize, ysize): + xstart = xstart - 1 + ystart = ystart - 1 + mask_array[ystart:ystart + ysize, xstart:xstart + xsize] = subarray_index + +def test_parse_subarray_map(): + subarray_maps_metadata = [ + {"xstart" : 80, "ystart" : 70, "xsize" : 10, "ysize" : 10}, + {"xstart" : 10, "ystart" : 20, "xsize" : 20, "ysize" : 20} + ] + + subarr_map = np.zeros((100,100), dtype=np.int16) + for i, shape in enumerate(subarray_maps_metadata): + set_subarray_mask(subarr_map, subarray_index=i+1, **shape) + + parse_subarray_map_output = parse_subarray_map(subarr_map) + + # Test parse_subarray_map + assert subarray_maps_metadata == parse_subarray_map_output + + # Create toy data + image = liger_iris_pipeline.datamodels.LigerIrisImageModel(data=np.zeros((100, 100))) + image.dq[25, 25] = 16 + image.dq[26, 26] = 1 + image["subarr_map"] = subarr_map + + # Test the step class + step = ParseSubarrayMapStep() + output = step.run(image) + + # Test + for each_parsed, each_input in zip(output.subarr_map, subarray_maps_metadata): + assert each_parsed.instance == each_input + + # If a pixel is already flagged as subarray, don't mess it up + assert output.dq[25, 25] == 16 + + # conserve existing flags + assert output.dq[26, 26] == 17 + + # Test + np.testing.assert_array_equal( + np.bitwise_and(output.dq, int(2**4)) > 0, + subarray_map != 0 + ) + + +test_parse_subarray_map() \ No newline at end of file From 390bafb34a7d7df70580ad253dd6f0cbbac146c1 Mon Sep 17 00:00:00 2001 From: Andrea Zonca Date: Fri, 5 Jul 2024 09:55:13 -0700 Subject: [PATCH 2/3] fix: recover lost subarray_map metadata field --- .../schemas/liger_iris_core.schema.yaml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/liger_iris_pipeline/datamodels/schemas/liger_iris_core.schema.yaml b/liger_iris_pipeline/datamodels/schemas/liger_iris_core.schema.yaml index 4f857e7..806ecb1 100644 --- a/liger_iris_pipeline/datamodels/schemas/liger_iris_core.schema.yaml +++ b/liger_iris_pipeline/datamodels/schemas/liger_iris_core.schema.yaml @@ -677,4 +677,22 @@ properties: title: Number of pixels in detector y-axis direction type: integer fits_keyword: DETYSIZ - blend_table: True \ No newline at end of file + blend_table: True + subarray_map: + title: Subarray map + type: array + items: + type: object + properties: + xstart: + title: Starting pixel in axis 1 direction + type: integer + ystart: + title: Starting pixel in axis 2 direction + type: integer + xsize: + title: Number of pixels in axis 1 direction + type: integer + ysize: + title: Number of pixels in axis 2 direction + type: integer From 3ca065b1bc3922e552430585e2ffae68aa5eb5cd Mon Sep 17 00:00:00 2001 From: Bryson Cale Date: Mon, 8 Jul 2024 09:08:19 -0700 Subject: [PATCH 3/3] fix: refactoring of parse_subarray_map_step and test --- .../parse_subarray_map_step.py | 4 +- .../tests/test_parse_subarray_map.ipynb | 259 ------------------ .../tests/test_parse_subarray_map.py | 25 +- 3 files changed, 16 insertions(+), 272 deletions(-) delete mode 100644 liger_iris_pipeline/tests/test_parse_subarray_map.ipynb diff --git a/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py b/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py index db324c6..f81f3f5 100644 --- a/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py +++ b/liger_iris_pipeline/parse_subarray_map/parse_subarray_map_step.py @@ -8,7 +8,7 @@ SUBARRAY_DQ_BIT = 4 - +# NOTE: xstart/ystart use 1-based indexing def parse_subarray_map(subarray_map): subarray_metadata = [] for subarray_id in range(1, 10 + 1): @@ -46,9 +46,11 @@ def process(self, input): result = input_model.copy() + # Create metadata from image ID map for each in parse_subarray_map(result["subarr_map"]): result.meta.subarray_map.append(each) + # Indicate subarrays in dq flags result.dq[result["subarr_map"] != 0] = np.bitwise_or( result.dq[result["subarr_map"] != 0], 2 ** SUBARRAY_DQ_BIT diff --git a/liger_iris_pipeline/tests/test_parse_subarray_map.ipynb b/liger_iris_pipeline/tests/test_parse_subarray_map.ipynb deleted file mode 100644 index 67583d1..0000000 --- a/liger_iris_pipeline/tests/test_parse_subarray_map.ipynb +++ /dev/null @@ -1,259 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import iris_pipeline\n", - "iris_pipeline.monkeypatch_jwst_datamodels()\n", - "\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "iris_pipeline.__file__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test the `parse_subarray_map` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from iris_pipeline.parse_subarray_map.parse_subarray_map_step import parse_subarray_map" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "subarray_maps_metadata = []\n", - "subarray_maps_metadata.append({\"xstart\":80, \"ystart\":70, \"xsize\":10, \"ysize\":10}) \n", - "subarray_maps_metadata.append({\"xstart\":10, \"ystart\":20, \"xsize\":20, \"ysize\":20}) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def set_subarray_mask(mask_array, subarray_index, xstart, ystart, xsize, ysize):\n", - " xstart = xstart - 1\n", - " ystart = ystart - 1\n", - " mask_array[ystart:ystart+ysize, xstart:xstart+xsize] = subarray_index " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "subarray_map = np.zeros((100,100), dtype=np.int16)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i, shape in enumerate(subarray_maps_metadata):\n", - " set_subarray_mask(subarray_map, subarray_index=i+1, **shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.imshow(subarray_map);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "parse_subarray_map_output = parse_subarray_map(subarray_map)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "type(parse_subarray_map_output[0][\"xstart\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert subarray_maps_metadata == parse_subarray_map_output" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test the pipeline step" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from iris_pipeline import ParseSubarrayMapStep" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "im = iris_pipeline.datamodels.IRISImageModel(data=np.zeros((100,100)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "im.dq[25, 25] = 16" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "im.dq[26, 26] = 1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "im[\"subarr_map\"] = subarray_map" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "output = ParseSubarrayMapStep().run(im)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for each_parsed, each_input in zip(output.meta.subarray_map, subarray_maps_metadata):\n", - " assert each_parsed.instance == each_input" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# if a pixel is already flagged as subarray, don't mess it up\n", - "assert output.dq[25, 25] == 16" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# conserve existing flags\n", - "assert output.dq[26, 26] == 17" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.imshow(output.dq)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.testing.assert_array_equal(np.bitwise_and(output.dq, int(2**4)) > 0, subarray_map != 0)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "jwst", - "language": "python", - "name": "jwst" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/liger_iris_pipeline/tests/test_parse_subarray_map.py b/liger_iris_pipeline/tests/test_parse_subarray_map.py index 57df543..bbd765d 100644 --- a/liger_iris_pipeline/tests/test_parse_subarray_map.py +++ b/liger_iris_pipeline/tests/test_parse_subarray_map.py @@ -5,27 +5,31 @@ from liger_iris_pipeline import ParseSubarrayMapStep import numpy as np + def set_subarray_mask(mask_array, subarray_index, xstart, ystart, xsize, ysize): xstart = xstart - 1 ystart = ystart - 1 mask_array[ystart:ystart + ysize, xstart:xstart + xsize] = subarray_index + def test_parse_subarray_map(): + + + # Define simple subarray metadata and image ID map + # ID is just the 1-based index in this list (1, 2, ...) subarray_maps_metadata = [ {"xstart" : 80, "ystart" : 70, "xsize" : 10, "ysize" : 10}, {"xstart" : 10, "ystart" : 20, "xsize" : 20, "ysize" : 20} ] - subarr_map = np.zeros((100,100), dtype=np.int16) for i, shape in enumerate(subarray_maps_metadata): set_subarray_mask(subarr_map, subarray_index=i+1, **shape) + # Test parse_subarray_map function parse_subarray_map_output = parse_subarray_map(subarr_map) - - # Test parse_subarray_map assert subarray_maps_metadata == parse_subarray_map_output - # Create toy data + # Create toy Image object with these subarrays image = liger_iris_pipeline.datamodels.LigerIrisImageModel(data=np.zeros((100, 100))) image.dq[25, 25] = 16 image.dq[26, 26] = 1 @@ -35,8 +39,8 @@ def test_parse_subarray_map(): step = ParseSubarrayMapStep() output = step.run(image) - # Test - for each_parsed, each_input in zip(output.subarr_map, subarray_maps_metadata): + # Test each parsed subarray map is equal to what we defined above. + for each_parsed, each_input in zip(output.meta.subarray_map, subarray_maps_metadata): assert each_parsed.instance == each_input # If a pixel is already flagged as subarray, don't mess it up @@ -45,11 +49,8 @@ def test_parse_subarray_map(): # conserve existing flags assert output.dq[26, 26] == 17 - # Test + # Test DQ flags np.testing.assert_array_equal( np.bitwise_and(output.dq, int(2**4)) > 0, - subarray_map != 0 - ) - - -test_parse_subarray_map() \ No newline at end of file + subarr_map != 0 + ) \ No newline at end of file