From fe86286a59c4c40f2ef35d8565c68170fd77fe8f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 16 Jul 2024 06:58:34 +0000 Subject: [PATCH 01/28] DSEGOG-269 Add config section for waveforms - This commit separates out the image and waveform thumbnail config into separate sections - Also adds a `line_width` config option --- .github/ci_config.yml | 6 ++++-- operationsgateway_api/config.yml.example | 6 ++++-- operationsgateway_api/src/config.py | 9 +++++++-- operationsgateway_api/src/records/image.py | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/ci_config.yml b/.github/ci_config.yml index c761e38f..13033195 100644 --- a/.github/ci_config.yml +++ b/.github/ci_config.yml @@ -4,12 +4,14 @@ app: # API will auto-reload when changes on code files are detected reload: true images: - image_thumbnail_size: [50, 50] - waveform_thumbnail_size: [100, 100] + thumbnail_size: [50, 50] default_colour_map: viridis colourbar_height_pixels: 16 upload_image_threads: 4 preferred_colour_map_pref_name: PREFERRED_COLOUR_MAP +waveforms: + thumbnail_size: [100, 100] + line_width: 0.3 echo: url: http://127.0.0.1:9000 username: operationsgateway diff --git a/operationsgateway_api/config.yml.example b/operationsgateway_api/config.yml.example index f9916b45..bae2838c 100644 --- a/operationsgateway_api/config.yml.example +++ b/operationsgateway_api/config.yml.example @@ -7,13 +7,15 @@ app: reload: true images: # Thumbnail sizes should only ever be two element lists, of a x, y resolution - image_thumbnail_size: [50, 50] - waveform_thumbnail_size: [100, 100] + thumbnail_size: [50, 50] # Colour maps for Matplotlib can be used default_colour_map: viridis colourbar_height_pixels: 16 upload_image_threads: 4 preferred_colour_map_pref_name: PREFERRED_COLOUR_MAP +waveforms: + thumbnail_size: [100, 100] + line_width: 0.3 echo: url: https://s3.echo.stfc.ac.uk username: username diff --git a/operationsgateway_api/src/config.py b/operationsgateway_api/src/config.py index 27232e84..4114e9ca 100644 --- a/operationsgateway_api/src/config.py +++ b/operationsgateway_api/src/config.py @@ -27,8 +27,7 @@ class App(BaseModel): class ImagesConfig(BaseModel): # The dimensions will be stored as a list in the YAML file, but are cast to tuple # using `typing.Tuple` because this is the type used by Pillow - image_thumbnail_size: Tuple[int, int] - waveform_thumbnail_size: Tuple[int, int] + thumbnail_size: Tuple[int, int] # the system default colour map (used if no user preference is set) default_colour_map: StrictStr colourbar_height_pixels: StrictInt @@ -36,6 +35,11 @@ class ImagesConfig(BaseModel): preferred_colour_map_pref_name: StrictStr +class WaveformsConfig(BaseModel): + thumbnail_size: Tuple[int, int] + line_width: float + + class EchoConfig(BaseModel): url: StrictStr username: StrictStr @@ -107,6 +111,7 @@ class APIConfig(BaseModel): auth: AuthConfig experiments: ExperimentsConfig images: ImagesConfig + waveforms: WaveformsConfig echo: EchoConfig export: ExportConfig diff --git a/operationsgateway_api/src/records/image.py b/operationsgateway_api/src/records/image.py index 754688d3..a24d5e74 100644 --- a/operationsgateway_api/src/records/image.py +++ b/operationsgateway_api/src/records/image.py @@ -53,7 +53,7 @@ def create_thumbnail(self) -> None: if img.mode == "I;16": img = img.convert("I") - img.thumbnail(Config.config.images.image_thumbnail_size) + img.thumbnail(Config.config.images.thumbnail_size) # convert 16 bit greyscale thumbnails to 8 bit to save space if img.mode == "I": log.debug("Converting 16 bit greyscale thumbnail to 8 bit") From 7d3032c0979f5239232ee718ff919b3374b61e83 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 16 Jul 2024 07:12:50 +0000 Subject: [PATCH 02/28] DSEGOG-269 Make waveform thumbnail sizes configurable - Also added comments throughout `_create_thumbnail_plot()` to show what each part is doing --- operationsgateway_api/src/records/waveform.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/operationsgateway_api/src/records/waveform.py b/operationsgateway_api/src/records/waveform.py index 530068b6..a6e6ab50 100644 --- a/operationsgateway_api/src/records/waveform.py +++ b/operationsgateway_api/src/records/waveform.py @@ -9,6 +9,7 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: I202 +from operationsgateway_api.src.config import Config from operationsgateway_api.src.exceptions import EchoS3Error, WaveformError from operationsgateway_api.src.models import WaveformModel from operationsgateway_api.src.records.echo_interface import EchoInterface @@ -77,19 +78,25 @@ def _create_thumbnail_plot(self, buffer) -> None: Using Matplotlib, create a thumbnail sized plot of the waveform data and save it to a bytes IO object provided as a parameter to this function """ - # Making changes to plot so figure size and line width is correct and axes are - # disabled - plt.figure(figsize=(1, 0.75)) + thumbnail_size = Config.config.waveforms.thumbnail_size + # 1 in figsize = 100px + plt.figure(figsize=(thumbnail_size[0] / 100, thumbnail_size[1] / 100)) + # Removes the notches on the plot that provide a scale plt.xticks([]) plt.yticks([]) + # Line width is configurable - thickness of line in the waveform plt.plot( self.waveform.x, self.waveform.y, - linewidth=0.5, + linewidth=Config.config.waveforms.line_width, ) + # Disables all axis decorations plt.axis("off") + # Removes the frame around the plot plt.box(False) + # Setting bbox_inches="tight" and pad_inches=0 removes padding around figure + # to make best use of the limited pixels available in a thumbnail plt.savefig(buffer, format="PNG", bbox_inches="tight", pad_inches=0, dpi=130) # Flushes the plot to remove data from previously ingested waveforms plt.clf() From 313d20f9a47551f3d02383dfd461849dc48faa05 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 16 Jul 2024 07:59:31 +0000 Subject: [PATCH 03/28] DSEGOG-269 Add tests for image and waveform thumbnail sizes --- test/records/test_image.py | 35 +++++++++++++++++++++++++++++++++++ test/records/test_waveform.py | 26 ++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/test/records/test_image.py b/test/records/test_image.py index 8173cac7..4fc73855 100644 --- a/test/records/test_image.py +++ b/test/records/test_image.py @@ -63,6 +63,41 @@ def test_create_thumbnail(self): # the reason only 0s are being asserted is because this is checking the hash of # a purely black 300x300 square created in the test_image above + @pytest.mark.parametrize( + "image_size, config_thumbnail_size, expected_thumbnail_size", + [ + pytest.param((300, 300), (50, 50), (50, 50), id="50x50 thumbnail"), + pytest.param((400, 300), (60, 80), (60, 80), id="60x80 thumbnail"), + pytest.param( + (300, 300), + (75, 100), + (75, 75), + id="75x100 thumbnail (square image)", + ), + ], + ) + def test_create_thumbnail_config_size( + self, + image_size, + config_thumbnail_size, + expected_thumbnail_size, + ): + test_image = Image( + ImageModel( + path="test/path/photo.png", + data=np.ones(shape=image_size, dtype=np.int8), + ), + ) + with patch( + "operationsgateway_api.src.config.Config.config.images.thumbnail_size", + config_thumbnail_size, + ): + test_image.create_thumbnail() + + bytes_thumbnail = base64.b64decode(test_image.thumbnail) + img = PILImage.open(BytesIO(bytes_thumbnail)) + assert img.size == expected_thumbnail_size + def test_alternative_mode_thumbnail(self): test_image = Image( ImageModel( diff --git a/test/records/test_waveform.py b/test/records/test_waveform.py index ec216714..f65cc880 100644 --- a/test/records/test_waveform.py +++ b/test/records/test_waveform.py @@ -1,3 +1,8 @@ +import base64 +from io import BytesIO +from unittest.mock import patch + +from PIL import Image import pytest from operationsgateway_api.src.exceptions import WaveformError @@ -25,3 +30,24 @@ async def test_insert_waveform_success(self, remove_waveform_entry): async def test_waveform_not_found(self): with pytest.raises(WaveformError, match="Waveform could not be found"): Waveform.get_waveform("19520605070023/test-channel-name.json") + + @pytest.mark.parametrize( + "config_thumbnail_size", + [ + pytest.param((50, 50), id="50x50 thumbnail"), + pytest.param((60, 80), id="60x80 thumbnail"), + pytest.param((90, 40), id="90x40 thumbnail"), + pytest.param((75, 100), id="75x100 thumbnail"), + ], + ) + def test_create_thumbnail_plot_size(self, config_thumbnail_size): + test_waveform = Waveform(TestWaveform.test_waveform) + with patch( + "operationsgateway_api.src.config.Config.config.waveforms.thumbnail_size", + config_thumbnail_size, + ): + test_waveform.create_thumbnail() + + bytes_thumbnail = base64.b64decode(test_waveform.thumbnail) + img = Image.open(BytesIO(bytes_thumbnail)) + assert img.size == config_thumbnail_size From 7f4601279c5d982dafefd18bda3b8a4ee5bf10d6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 16 Jul 2024 08:02:15 +0000 Subject: [PATCH 04/28] DSEGOG-269 Move image tests to `test/images/` --- test/{records => images}/jet_image.png | Bin test/{records => images}/original_image.png | Bin .../test_false_colour_handler.py | 0 .../test_false_colour_image_errors.py | 0 test/{records => images}/test_image.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename test/{records => images}/jet_image.png (100%) rename test/{records => images}/original_image.png (100%) rename test/{records => images}/test_false_colour_handler.py (100%) rename test/{records => images}/test_false_colour_image_errors.py (100%) rename test/{records => images}/test_image.py (100%) diff --git a/test/records/jet_image.png b/test/images/jet_image.png similarity index 100% rename from test/records/jet_image.png rename to test/images/jet_image.png diff --git a/test/records/original_image.png b/test/images/original_image.png similarity index 100% rename from test/records/original_image.png rename to test/images/original_image.png diff --git a/test/records/test_false_colour_handler.py b/test/images/test_false_colour_handler.py similarity index 100% rename from test/records/test_false_colour_handler.py rename to test/images/test_false_colour_handler.py diff --git a/test/records/test_false_colour_image_errors.py b/test/images/test_false_colour_image_errors.py similarity index 100% rename from test/records/test_false_colour_image_errors.py rename to test/images/test_false_colour_image_errors.py diff --git a/test/records/test_image.py b/test/images/test_image.py similarity index 100% rename from test/records/test_image.py rename to test/images/test_image.py From af8803048d6b4adba9eec0198ce32327d0c89c89 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 16 Jul 2024 12:54:04 +0000 Subject: [PATCH 05/28] DSEGOG-269 Correct path for test image --- test/images/test_false_colour_image_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/images/test_false_colour_image_errors.py b/test/images/test_false_colour_image_errors.py index de9511d2..c90994f7 100644 --- a/test/images/test_false_colour_image_errors.py +++ b/test/images/test_false_colour_image_errors.py @@ -18,7 +18,7 @@ def test_invalid_bits_per_pixel(self): ) def test_invalid_image_mode(self): - image_path = "test/records/jet_image.png" + image_path = "test/images/jet_image.png" pil_image = Image.open(image_path).convert("RGB") with pytest.raises(ImageError, match="Image mode RGB not recognised"): From f211d433df72b799ba0db598b6ad0456f6fc2894 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 16 Jul 2024 13:52:55 +0000 Subject: [PATCH 06/28] DSEGOG-269 Update hash values for waveform thumbnails in channel summary test - This needs to be done because the default line width has changed, therefore the thumbnails look slightly different --- test/endpoints/test_channel_summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/endpoints/test_channel_summary.py b/test/endpoints/test_channel_summary.py index 4f9c757b..e97e0801 100644 --- a/test/endpoints/test_channel_summary.py +++ b/test/endpoints/test_channel_summary.py @@ -83,8 +83,8 @@ def test_valid_scalar_channel_summary( "most_recent_date": "2023-06-05T16:00:00", "recent_sample": [ {"2023-06-05T16:00:00": "e6e41be41be19a19"}, - {"2023-06-05T15:00:00": "e6e61be51ae11a19"}, - {"2023-06-05T14:00:00": "e6e41be51ae1981b"}, + {"2023-06-05T15:00:00": "e6e61be41ae11a39"}, + {"2023-06-05T14:00:00": "e6e41be51ae19a19"}, ], }, False, From b4b977807b5ac91b26670d11b6ea032658738863 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 17 Jul 2024 11:18:22 +0000 Subject: [PATCH 07/28] DSEGOG-337 Add except for common error if `self.url` doesn't contain protocol --- util/realistic_data/ingest/api_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/util/realistic_data/ingest/api_client.py b/util/realistic_data/ingest/api_client.py index 5fb4729f..a41cc5f0 100644 --- a/util/realistic_data/ingest/api_client.py +++ b/util/realistic_data/ingest/api_client.py @@ -44,6 +44,11 @@ def login(self) -> str: return access_token, refresh_token except ConnectionError: print(f"Cannot connect with API at {self.url} for {endpoint}") + except requests.exceptions.InvalidSchema: + print( + "Invalid schema when logging in with requests. Have you added" + " http/https to the start of the API URL?", + ) def refresh(self) -> str: print(f"Refresh token as '{Config.config.api.username}'") From 8d7a472abc5e37c16a8437ba47aa693b6fd71a2a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 17 Jul 2024 11:19:49 +0000 Subject: [PATCH 08/28] DSEGOG-337 Add except when a file doesn't exist --- util/realistic_data/daily_ingestor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/util/realistic_data/daily_ingestor.py b/util/realistic_data/daily_ingestor.py index f6d9438b..a8295f21 100644 --- a/util/realistic_data/daily_ingestor.py +++ b/util/realistic_data/daily_ingestor.py @@ -55,8 +55,12 @@ def main(): f" Response: {response_code}. Moving this file to so it can be" f" investigated by a human: {failed_ingests_directory}", ) - - file_to_ingest.rename(f"{failed_ingests_directory}/{file_to_ingest.name}") + try: + file_to_ingest.rename( + f"{failed_ingests_directory}/{file_to_ingest.name}", + ) + except FileNotFoundError as e: + print(e) if __name__ == "__main__": From 2831c005116bd88ed76fd4eb74e30444917bd55a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 17 Jul 2024 11:20:32 +0000 Subject: [PATCH 09/28] DSEGOG-337 Move files to be ingested to an 'in progress' directory --- util/realistic_data/daily_ingestor.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/util/realistic_data/daily_ingestor.py b/util/realistic_data/daily_ingestor.py index a8295f21..246f4f5c 100644 --- a/util/realistic_data/daily_ingestor.py +++ b/util/realistic_data/daily_ingestor.py @@ -28,11 +28,25 @@ def main(): tzinfo=tz.gettz("Europe/London"), ) for file in hdf_files + # Removes `in_progress/` directory from list of files + if file.is_file() } + files_to_ingest = [] for file_path, file_datetime in hdf_file_dates.items(): if file_datetime < datetime.now(tz=tz.gettz("Europe/London")): - files_to_ingest.append(file_path) + # Move the file to an 'in progress' directory to avoid collisions with other + # instances of this script (i.e. cron job every minute) + try: + # Updating `file_path` so the change in directory is captured when the + # file is opened later in the script + file_path = file_path.rename( + f"{hdf_data_directory}/in_progress/{file_path.name}", + ) + except FileNotFoundError as e: + print(e) + else: + files_to_ingest.append(file_path) print("Files to ingest:") pprint([file.name for file in files_to_ingest]) From f14b5ddb3ec7fd5a80bd12bc1981f97ce14acd27 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 17 Jul 2024 11:21:00 +0000 Subject: [PATCH 10/28] DSEGOG-337 Move any left over files back to the data directory --- util/realistic_data/daily_ingestor.py | 54 +++++++++++++++++---------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/util/realistic_data/daily_ingestor.py b/util/realistic_data/daily_ingestor.py index 246f4f5c..524390ac 100644 --- a/util/realistic_data/daily_ingestor.py +++ b/util/realistic_data/daily_ingestor.py @@ -51,30 +51,46 @@ def main(): print("Files to ingest:") pprint([file.name for file in files_to_ingest]) - og_api = APIClient(og_api_url) + try: + og_api = APIClient(og_api_url) - for file_to_ingest in files_to_ingest: - with open(file_to_ingest, "rb") as hdf_file: - response_code = og_api.submit_hdf({file_to_ingest.name: hdf_file}) + for file_to_ingest in files_to_ingest: + with open(file_to_ingest, "rb") as hdf_file: + response_code = og_api.submit_hdf({file_to_ingest.name: hdf_file}) - if response_code == 201 or response_code == 200: - print( - f"Successfully ingested ({response_code}) '{file_to_ingest.name}'." - " HDF file will be deleted", - ) - file_to_ingest.unlink(missing_ok=True) - else: - print( - f"Ingestion unsuccessful for: {file_to_ingest.name}," - f" Response: {response_code}. Moving this file to so it can be" - f" investigated by a human: {failed_ingests_directory}", - ) - try: - file_to_ingest.rename( - f"{failed_ingests_directory}/{file_to_ingest.name}", + if response_code == 201 or response_code == 200: + print( + f"Successfully ingested ({response_code}) '{file_to_ingest.name}'." + " HDF file will be deleted", + ) + file_to_ingest.unlink(missing_ok=True) + else: + print( + f"Ingestion unsuccessful for: {file_to_ingest.name}," + f" Response: {response_code}. Moving this file to so it can be" + f" investigated by a human: {failed_ingests_directory}", ) + try: + file_to_ingest.rename( + f"{failed_ingests_directory}/{file_to_ingest.name}", + ) + except FileNotFoundError as e: + print(e) + + except Exception as e: + print(e) + finally: + # For any files that are left in `in_progress/` due to an issue (e.g. login + # failure) move them back to the original directory to be picked up by a future + # execution of this script + in_progress_files = Path(f"{hdf_data_directory}/in_progress").iterdir() + for file in in_progress_files: + try: + file.rename(f"{hdf_data_directory}/{file.name}") except FileNotFoundError as e: print(e) + else: + print(f"{file} moved back to data directory") if __name__ == "__main__": From 650e2a354088d9a8db60fab053f8ee1ff6ec02a4 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 17 Jul 2024 12:31:30 +0000 Subject: [PATCH 11/28] DSEGOG-337 Create an 'in progress' directory for each script execution --- util/realistic_data/daily_ingestor.py | 28 ++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/util/realistic_data/daily_ingestor.py b/util/realistic_data/daily_ingestor.py index 524390ac..e59279af 100644 --- a/util/realistic_data/daily_ingestor.py +++ b/util/realistic_data/daily_ingestor.py @@ -28,10 +28,21 @@ def main(): tzinfo=tz.gettz("Europe/London"), ) for file in hdf_files - # Removes `in_progress/` directory from list of files + # Includes files only, any directories are ignored if file.is_file() } + in_progress_date_format = f"{hdf_file_date_format}%S" + current_datetime = ( + datetime.now().replace(microsecond=0).strftime(in_progress_date_format) + ) + Path(f"{hdf_data_directory}/in_progress_{current_datetime}").mkdir( + parents=True, + exist_ok=True, + ) + in_progress_dir = Path(f"{hdf_data_directory}/in_progress_{current_datetime}") + print(f"Created {in_progress_dir} directory") + files_to_ingest = [] for file_path, file_datetime in hdf_file_dates.items(): if file_datetime < datetime.now(tz=tz.gettz("Europe/London")): @@ -41,7 +52,7 @@ def main(): # Updating `file_path` so the change in directory is captured when the # file is opened later in the script file_path = file_path.rename( - f"{hdf_data_directory}/in_progress/{file_path.name}", + f"{in_progress_dir}/{file_path.name}", ) except FileNotFoundError as e: print(e) @@ -76,14 +87,13 @@ def main(): ) except FileNotFoundError as e: print(e) - except Exception as e: print(e) finally: - # For any files that are left in `in_progress/` due to an issue (e.g. login - # failure) move them back to the original directory to be picked up by a future - # execution of this script - in_progress_files = Path(f"{hdf_data_directory}/in_progress").iterdir() + # For any files that are left in the 'in progress' directory (perhaps due to an + # issue such as login failure) move them back to the original directory to be + # picked up by a future execution of this script + in_progress_files = Path(in_progress_dir).iterdir() for file in in_progress_files: try: file.rename(f"{hdf_data_directory}/{file.name}") @@ -92,6 +102,10 @@ def main(): else: print(f"{file} moved back to data directory") + # Clean up the 'in progress' directory + print(f"Going to remove: {in_progress_dir}") + Path.rmdir(in_progress_dir) + if __name__ == "__main__": main() From e0bdd09b52bbfb2c28f443a3ac29f664315e137c Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 17 Jul 2024 15:22:22 +0000 Subject: [PATCH 12/28] DSEGOG-327 Add `url_prefix` to API and use that for root path - This will fix `/docs` where the API is using a reverse proxy that isn't `/` --- .github/ci_config.yml | 1 + operationsgateway_api/config.yml.example | 1 + operationsgateway_api/src/config.py | 3 ++- operationsgateway_api/src/main.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/ci_config.yml b/.github/ci_config.yml index c761e38f..48d62904 100644 --- a/.github/ci_config.yml +++ b/.github/ci_config.yml @@ -3,6 +3,7 @@ app: port: 8000 # API will auto-reload when changes on code files are detected reload: true + url_prefix: "" images: image_thumbnail_size: [50, 50] waveform_thumbnail_size: [100, 100] diff --git a/operationsgateway_api/config.yml.example b/operationsgateway_api/config.yml.example index f9916b45..b19e0792 100644 --- a/operationsgateway_api/config.yml.example +++ b/operationsgateway_api/config.yml.example @@ -5,6 +5,7 @@ app: port: 8000 # API will auto-reload when changes on code files are detected reload: true + url_prefix: "" images: # Thumbnail sizes should only ever be two element lists, of a x, y resolution image_thumbnail_size: [50, 50] diff --git a/operationsgateway_api/src/config.py b/operationsgateway_api/src/config.py index 27232e84..0a6a1d87 100644 --- a/operationsgateway_api/src/config.py +++ b/operationsgateway_api/src/config.py @@ -22,6 +22,7 @@ class App(BaseModel): host: Optional[StrictStr] = None port: Optional[StrictInt] = None reload: Optional[StrictBool] = None + url_prefix: StrictStr class ImagesConfig(BaseModel): @@ -102,7 +103,7 @@ class APIConfig(BaseModel): # When in production, there's no `app` section in the config file. A default value # (i.e. an empty instance of `App`) has been assigned so that if the code attempts # to access a config value in this section, an error is prevented - app: Optional[App] = App() + app: App mongodb: MongoDB auth: AuthConfig experiments: ExperimentsConfig diff --git a/operationsgateway_api/src/main.py b/operationsgateway_api/src/main.py index 99769771..beb9bc49 100644 --- a/operationsgateway_api/src/main.py +++ b/operationsgateway_api/src/main.py @@ -76,6 +76,7 @@ async def get_experiments_on_startup(): description=api_description, default_response_class=ORJSONResponse, lifespan=lifespan, + root_path=Config.config.app.url_prefix, ) app.add_middleware( From 31c82c1915ef5fa2cf1587c388ea3ac2102c30b8 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 18 Jul 2024 10:15:36 +0000 Subject: [PATCH 13/28] DSEGOG-283 Fix Pydantic class-based config warning --- operationsgateway_api/src/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/operationsgateway_api/src/models.py b/operationsgateway_api/src/models.py index 9020a428..8353a0e5 100644 --- a/operationsgateway_api/src/models.py +++ b/operationsgateway_api/src/models.py @@ -49,9 +49,7 @@ class WaveformModel(BaseModel): path: Optional[str] = default_exclude_field x: Optional[Union[List[float], Any]] y: Optional[Union[List[float], Any]] - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) @field_validator("x", "y", mode="before") def encode_values(cls, value): # noqa: N805 From 4d0fd0934b5045892b7f83348473cbfb4b965a84 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 18 Jul 2024 10:16:17 +0000 Subject: [PATCH 14/28] DSEGOG-283 Fix warning from `numpy` about elementwise comparsion failure --- test/records/ingestion/create_test_hdf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/records/ingestion/create_test_hdf.py b/test/records/ingestion/create_test_hdf.py index 08d48112..cda618d6 100644 --- a/test/records/ingestion/create_test_hdf.py +++ b/test/records/ingestion/create_test_hdf.py @@ -127,7 +127,7 @@ async def create_test_hdf_file( # noqa: C901 pm_201_fe_en.attrs.create("channel_dtype", "scalar") if required_attributes and "scalar" in required_attributes: scalar = required_attributes["scalar"] - if scalar["data"] != "missing": + if not isinstance(scalar["data"], str) or scalar["data"] != "missing": pm_201_fe_en.create_dataset("data", data=(scalar["data"])) else: pm_201_fe_en.create_dataset("data", data=366272) @@ -285,7 +285,7 @@ async def create_test_hdf_file( # noqa: C901 data = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.uint16) if required_attributes and "image" in required_attributes: image = required_attributes["image"] - if image["data"] != "missing": + if not isinstance(image["data"], str) or image["data"] != "missing": pm_201_fe_cam_1.create_dataset("data", data=(image["data"])) else: pm_201_fe_cam_1.create_dataset("data", data=data) @@ -373,12 +373,12 @@ async def create_test_hdf_file( # noqa: C901 if required_attributes and "waveform" in required_attributes: waveform = required_attributes["waveform"] if "x" in waveform: - if waveform["x"] != "missing": + if not isinstance(waveform["x"], str) or waveform["x"] != "missing": pm_201_hj_pd.create_dataset("x", data=(waveform["x"])) else: pm_201_hj_pd.create_dataset("x", data=x) if "y" in waveform: - if waveform["y"] != "missing": + if not isinstance(waveform["y"], str) or waveform["y"] != "missing": pm_201_hj_pd.create_dataset("y", data=(waveform["y"])) else: pm_201_hj_pd.create_dataset("y", data=y) From 0c04f1967784c1912d4dc7ddea582031d52335b5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 19 Jul 2024 13:20:41 +0000 Subject: [PATCH 15/28] DSEGOG-338 Move Echo documentation to specific file --- README.md | 36 ------------------------------------ docs/echo_object_storage.md | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 36 deletions(-) create mode 100644 docs/echo_object_storage.md diff --git a/README.md b/README.md index e420fde8..d0c95a8b 100644 --- a/README.md +++ b/README.md @@ -66,39 +66,3 @@ mongoimport --db='opsgateway' --collection='users' --mode='upsert' --file='util/ ``` Using the `upsert` mode allows you to update existing users with any changes that are made (e.g. added an authorised route to their entry) and any new users are inserted as normal. The command's output states the number of documents that have been added and how many have been updated. - -## Echo Object Storage -Waveforms and images are stored using S3 object storage (using the same bucket), currently the STFC Echo instance. Lots of documentation online references the AWS offering, but as S3 is the underlying storage technique, we can interact with Echo in the same way that a user would interact with AWS S3. - -Configuration to connect with Echo is stored in the `echo` section of the config file - credentials are stored in Keeper. This section includes a bucket name, which is the location on S3 storage where images & waveforms will be stored. For the API, we have multiple buckets, used for different purposes. For example, there's a bucket used for the dev server, a bucket per developer for their development environment, as well as buckets that are created for a short period of time for specific testing. This ensures that we're not overwriting each other's data and causing issues. For GitHub Actions, each run will create a new bucket, ingest data for testing and delete the bucket at the end of the run. - -To manage buckets, [s4cmd](https://github.com/bloomreach/s4cmd) is a good command line utility. It provides an Unix-like interface to S3 storage, based off of `s3cmd` but has higher performance when interacting with large files. It is a development dependency for this repository but can also be installed using `pip`. There's an example configuration file in `.github/ci_s3cfg` which can be placed in `~/.s3cfg` and used for your own development environment. - -Here's a few useful example commands (the [s4cmd README](https://github.com/bloomreach/s4cmd/blob/master/README.md) provides useful information about all available commands): -```bash -# To make calling `s4cmd` easier when installed as a development dependency, I've added the following alias to `~/.bashrc` -# Change the path to the Poetry virtualenv as needed -alias s4cmd='/root/.cache/pypoetry/virtualenvs/operationsgateway-api-pfN98gKB-py3.8/bin/s4cmd --endpoint-url https://s3.echo.stfc.ac.uk' - -# The following commands assume the alias has been made -# Create a bucket called 'og-my-test-bucket' on STFC's Echo S3 -s4cmd mb s3://og-my-test-bucket - -# List everything that the current user can see -s4cmd ls - -# List everything inside 'og-my-test-bucket' -s4cmd ls s3://og-my-test-bucket - -# Remove all objects in bucket -s4cmd del --recursive s3://og-my-test-bucket -``` - -## API Startup -To start the API, use the following command: - -```bash -poetry run python -m operationsgateway_api.src.main -``` - -Assuming default configuration, the API will exist on 127.0.0.1:8000. You can visit `/docs` in a browser which will give an OpenAPI interface detailing each of the endpoints and an option to send requests to them. Alternatively, you can send requests to the API using a platform such as Postman to construct and save specific requests. diff --git a/docs/echo_object_storage.md b/docs/echo_object_storage.md new file mode 100644 index 00000000..a0397953 --- /dev/null +++ b/docs/echo_object_storage.md @@ -0,0 +1,35 @@ +# Echo Object Storage +Waveforms and images are stored using S3 object storage (using the same bucket), currently the STFC Echo instance. Lots of documentation online references the AWS offering, but as S3 is the underlying storage technique, we can interact with Echo in the same way that a user would interact with AWS S3. + +Configuration to connect with Echo is stored in the `echo` section of the config file - credentials are stored in Keeper. This section includes a bucket name, which is the location on S3 storage where images & waveforms will be stored. For the API, we have multiple buckets, used for different purposes. For example, there's a bucket used for the dev server, a bucket per developer for their development environment, as well as buckets that are created for a short period of time for specific testing. This ensures that we're not overwriting each other's data and causing issues. For GitHub Actions, each run will create a new bucket, ingest data for testing and delete the bucket at the end of the run. + +To manage buckets, [s4cmd](https://github.com/bloomreach/s4cmd) is a good command line utility. It provides an Unix-like interface to S3 storage, based off of `s3cmd` but has higher performance when interacting with large files. It is a development dependency for this repository but can also be installed using `pip`. There's an example configuration file in `.github/ci_s3cfg` which can be placed in `~/.s3cfg` and used for your own development environment. + +Here's a few useful example commands (the [s4cmd README](https://github.com/bloomreach/s4cmd/blob/master/README.md) provides useful information about all available commands): +```bash +# To make calling `s4cmd` easier when installed as a development dependency, I've added the following alias to `~/.bashrc` +# Change the path to the Poetry virtualenv as needed +alias s4cmd='/root/.cache/pypoetry/virtualenvs/operationsgateway-api-pfN98gKB-py3.8/bin/s4cmd --endpoint-url https://s3.echo.stfc.ac.uk' + +# The following commands assume the alias has been made +# Create a bucket called 'og-my-test-bucket' on STFC's Echo S3 +s4cmd mb s3://og-my-test-bucket + +# List everything that the current user can see +s4cmd ls + +# List everything inside 'og-my-test-bucket' +s4cmd ls s3://og-my-test-bucket + +# Remove all objects in bucket +s4cmd del --recursive s3://og-my-test-bucket +``` + +## API Startup +To start the API, use the following command: + +```bash +poetry run python -m operationsgateway_api.src.main +``` + +Assuming default configuration, the API will exist on 127.0.0.1:8000. You can visit `/docs` in a browser which will give an OpenAPI interface detailing each of the endpoints and an option to send requests to them. Alternatively, you can send requests to the API using a platform such as Postman to construct and save specific requests. From 4598e30fcccfc98519c17f5d95988906709b2420 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 19 Jul 2024 13:21:33 +0000 Subject: [PATCH 16/28] DSEGOG-338 Remove section in README about importing users - This isn't a process that needs to be performed by the developer, our ingest scripts import the test users for us --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index d0c95a8b..ca5a2090 100644 --- a/README.md +++ b/README.md @@ -54,15 +54,3 @@ Press enter twice when prompted for the password so as not to set one. The key should be OpenSSH encoded - this format is generated by default in Rocky 8. You can check whether your key is in the correct format by checking the start of the private key; it should be `-----BEGIN OPENSSH PRIVATE KEY-----`. Then edit the ```private_key_path``` and ```public_key_path``` settings in the ```auth``` section of the ```config.yml``` file to reflect the location where these keys have been created. - -### Adding User Accounts - -The authentication system requires any users of the system to have an account set up in the database. Two types of user login are currently supported: federal ID logins for "real" users, and "local" logins for functional accounts. - -To add some test accounts to the system, use the user data stored in `util/users_for_mongoimport.json`. Use the following command to import those users into the database: - -```bash -mongoimport --db='opsgateway' --collection='users' --mode='upsert' --file='util/users_for_mongoimport.json' -``` - -Using the `upsert` mode allows you to update existing users with any changes that are made (e.g. added an authorised route to their entry) and any new users are inserted as normal. The command's output states the number of documents that have been added and how many have been updated. From 7df0906f30d8c9acc9d56ad81dd268260a4fdabb Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 19 Jul 2024 13:23:42 +0000 Subject: [PATCH 17/28] DSEGOG-338 Move section about OpenAPI interface back to README --- README.md | 2 ++ docs/echo_object_storage.md | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca5a2090..6e36ad29 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ # OperationsGateway API This is an API built using [FastAPI](https://fastapi.tiangolo.com/) to work with [MongoDB](https://www.mongodb.com/) and the data stored as part of the OperationsGateway project. +Assuming default configuration, the API will exist on 127.0.0.1:8000. You can visit `/docs` in a browser which will give an OpenAPI interface detailing each of the endpoints and an option to send requests to them. Alternatively, you can send requests to the API using a platform such as Postman to construct and save specific requests. + ## Environment Setup If not already present, you may need to install development tools for the desired Python version using the appropriate package manager for your OS. For example, for Python3.8 on Fedora or RHEL: diff --git a/docs/echo_object_storage.md b/docs/echo_object_storage.md index a0397953..b36b8248 100644 --- a/docs/echo_object_storage.md +++ b/docs/echo_object_storage.md @@ -31,5 +31,3 @@ To start the API, use the following command: ```bash poetry run python -m operationsgateway_api.src.main ``` - -Assuming default configuration, the API will exist on 127.0.0.1:8000. You can visit `/docs` in a browser which will give an OpenAPI interface detailing each of the endpoints and an option to send requests to them. Alternatively, you can send requests to the API using a platform such as Postman to construct and save specific requests. From 24acfc7b5dff9fd1147e0cd4f5edae78185a36a5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 19 Jul 2024 13:31:53 +0000 Subject: [PATCH 18/28] DSEGOG-338 General updates to docs to make them more accurate --- docs/dev_server.md | 16 +++++++--------- docs/epac_simulated_data.md | 3 +-- docs/test_data.md | 17 ++++++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/dev_server.md b/docs/dev_server.md index 43ecb803..331ae958 100644 --- a/docs/dev_server.md +++ b/docs/dev_server.md @@ -24,23 +24,23 @@ To view that commit, you can go to GitHub and insert the commit hash into the fo To control the API on the dev server, a `systemd` service is added to the machine by Ansible. This allows you to do all the typical things you'd be able to do with a systemd service (i.e. start/stop/restart) and you can check on the logs using `journalctl -u og-api`. ## Apache/Certificates -The API is exposed to the outside world using a reverse proxy; the API lives on port 8000 but port 443 is used to access it. If a user accesses the API using port 80, it'll forward on their request to port 443. This works fine for GET requests but unusual things can happen for other request types (e.g. POST), particularly requests which contain a request body (see https://stackoverflow.com/a/21859641 for further information). The reverse proxy places the API on `/api`. This reserves `/` for the frontend, when it is deployed. +The API is exposed to the outside world using a reverse proxy; the API lives on port 8000 but port 443 is used to access it. If a user accesses the API using port 80, it'll forward on their request to port 443. This works fine for GET requests but unusual things can happen for other request types (e.g. POST), particularly requests which contain a request body (see https://stackoverflow.com/a/21859641 for further information). The reverse proxy places the API on `/api`. The frontend is deployed on `/`. Certificates are requested through the DI Service Desk, where the normal process applies - generate a CSR, submit a ticket containing the CSR asking for the certificate to be generated and download the files once they've been generated. Alan's [certificate cheatsheet](https://github.com/ral-facilities/dseg-docs/blob/master/certs-cheat-sheet.md) is a great resource to easily generate a CSR if you're not familiar with that process. When downloading the certificates, I click the following links: - `cert` - "as Certificate only, PEM encoded" -- `ca` (you need to remove the first certificate (our cert) from this file, so 2 remain) - "as Certificate (w/ issuer after), PEM encoded:" +- `ca` - "as Certificate (w/ issuer after), PEM encoded:" (you need to remove the first certificate (our cert) from this file, so 2 remain) The certificate files are stored in `/etc/httpd/certs/` and symlinks are applied to them which allows easy swapping of files. When changes are made, do a `systemctl restart httpd` to ensure any file/config changes take effect. -To open ports, use `firewall-cmd`; this is a Rocky 8 VM so this is different to older Centos 7 RIG VMs where `iptables` was used. To view current rules, use `firewall-cmd --list-all`. Ports 80 & 443 are opened when deployed using Ansible. +To open ports, use `firewall-cmd` (use `--permanent` to keep the rule persistent across reboots); this is a Rocky 8 VM so this is different to older Centos 7 RIG VMs where `iptables` was used. To view current rules, use `firewall-cmd --list-all`. Ports 80 & 443 are opened when deployed using Ansible. ## Storage -The API is hooked up to a MongoDB database provided by Database Services containing simulated data as well as using Echo S3. Credentials for these resources are stored in the shared Keeper folder and a specific bucket is used for the dev server (`s3://og-dev-server`). +The API is hooked up to a MongoDB database provided by Database Services containing simulated data as well as using Echo S3. Credentials for these resources are stored in the shared Keeper folder and a specific bucket is used for the dev server (`s3://OG-DEV-SERVER`). ### Simulated Data -The dev server contains 12 months worth of simulated data (October 2022-October 2023) which is reproducible using HDF files stored in the `s3://OG-YEAR-OF-SIMULATED-DATA` bucket in Echo. There are cron jobs which control data generated each day, a test to mimic incoming load from EPAC in production. More detail about the inner workings of this mechanism can be found in `docs/epac_simulated_data.md`. +The dev server contains 12 months worth of simulated data (October 2022-October 2023) which is reproducible using HDF files stored in the `s3://OG-YEAR-OF-SIMULATED-DATA` bucket in Echo. There are cron jobs which control data generated each day, functioning as a test to mimic incoming load from EPAC in production. More detail about the inner workings of this mechanism can be found in `docs/epac_simulated_data.md`. This ingestion was done using a separate cloud VM. It has its own instance of the API but is connected to the same database and Echo bucket as the dev server. To replicate this environment, run OperationsGateway Ansible using the `simulated-data-ingestion-vm` host. The `ingest_echo_data.py` script was run on that machine using the following command: ```bash @@ -49,11 +49,9 @@ This ingestion was done using a separate cloud VM. It has its own instance of th nohup $HOME/.local/bin/poetry run python -u util/realistic_data/ingest_echo_data.py >> /var/log/operationsgateway-api/echo_ingestion_script.log 2>&1 & ``` - ### Local Database for Gemini Data Before using simulated EPAC data, we used a small amount of Gemini data, stored in a local database; it is equivalent to the databases used in our development environments - local DBs, no auth, named `opsgateway`. There may be cases where in the future, we need to switch back to the Gemini data as this may allow us to test something that isn't so easy to test with the simulated data. To do this, the following things will need to be done: - Change the API config to point to the local database - both the URL and database name are different -- Point to a different Echo bucket - images were stored on disk when the Gemini data was last used so a new bucket should be created. The images used to be stored in `/epac_storage` but have since been deleted. +- Point to a different Echo bucket - images were stored on disk when the Gemini data was last used so a new bucket should be created (this might require running `ingest_hdf.py` to put the data onto Echo). The images used to be stored in `/epac_storage` but have since been deleted. +- Re-ingest the data using `ingest_hdf.py` (required as waveforms are now stored on Echo rather than in the database) - Restart the API using `systemctl restart og-api` - -An upcoming piece of work (as of Feb 2024) is to move waveforms to be stored in Echo (instead of the database). When this happens (and is deployed to the dev server), reingestion of the Gemini data might be required. Follow the instructions in `docs/test_data.md` for more info on this. diff --git a/docs/epac_simulated_data.md b/docs/epac_simulated_data.md index 943088f1..8a5b1178 100644 --- a/docs/epac_simulated_data.md +++ b/docs/epac_simulated_data.md @@ -1,6 +1,5 @@ # EPAC Simulated Data - -`util/realistic_data` is a directory that contains a number of scripts (and assisting Python code) to generate and ingest simulated data. This will allow us to have a more effective test & demo platform as the data should be closer to real data than the small amount of Gemini data we have access to. We hope to have one year of simulated data, and data generated each day to simulate an incoming load from the facility. +`util/realistic_data` is a directory that contains a number of scripts (and assisting Python code) to generate and ingest simulated data. This will allow us to have a more effective test & demo platform as the data should be closer to real data than the small amount of Gemini data we have access to. We have one year of simulated data, and data is generated & ingested each day to simulate an incoming load from the facility. The data is originally generated by a tool made by CLF, [EPAC-DataSim](https://github.com/CentralLaserFacility/EPAC-DataSim). The outputs from this tool are used in the scripts which form the following process: diff --git a/docs/test_data.md b/docs/test_data.md index cb6d670a..eb66437b 100644 --- a/docs/test_data.md +++ b/docs/test_data.md @@ -6,9 +6,9 @@ In OperationsGateway, there are two sources of test data: In the early stages of the API, a small amount of data was manually exported from the Gemini instance of eCat (using the system's export functionality) and converted into HDF files using [OG-HDF5](https://github.com/CentralLaserFacility/OG-HDF5). The files were stored in [operationsgateway-test-data](https://github.com/ral-facilities/operationsgateway-test-data) and contains a number of files of shot data as well as environmental. The README of that repo details how the files are separated into different directories and what each 'set' contains. -This was suitable as a starting point but wasn't entirely representative of the data from EPAC and the data volume wasn't large enough to test performance. The tests for this repo use simulated data as this data is more realistic but there may be times where the Gemini data is still relevant (e.g. going back to old branches). +This was suitable as a starting point but wasn't entirely representative of the data from EPAC and the data volume wasn't large enough to test performance. The API's tests now use simulated data as this data is more realistic. There may be cases where we need to revert back to the old Gemini data for manual testing on the dev server, `dev_server.md` has a section dedicated to explain how this would work (we haven't had a situation where we needed to revert back as of yet). -The simulated data are the HDF files generated by [EPAC-DataSim](https://github.com/CentralLaserFacility/EPAC-DataSim), a tool provided to us by CLF to give us data realistic to that of the data which we will receive from EPAC in production. As file sizes are large when the laser is firing (~300MB), only a few shot files are part of the simulated data used for tests and some environmental data too. This is a very small subset of the data stored on the dev server (i.e. the year of simulated data). +The simulated data are the HDF files generated by [EPAC-DataSim](https://github.com/CentralLaserFacility/EPAC-DataSim), a tool provided to us by CLF to give us data realistic to that of the data which we will receive from EPAC in production. The repo's README provides a good explanation of how to use it. The HDF files generated when there's an experiment are quite large (~300MB). As a result only a few shot files are part of the simulated data used for tests and some environmental data too - this helps keep CI times to a reasonable amount. This is a very small subset of the data stored on the dev server (i.e. the year of simulated data). ## Using Gemini Data To load Gemini data into your development environment, the `ingest_hdf.py` script needs to be used. The OperationsGateway Test Data repo also needs to be cloned on your machine beforehand. @@ -51,11 +51,14 @@ cp util/realistic_data/config.yml.example util/realistic_data/config.yml poetry run python util/realistic_data/ingest_echo_data.py ``` -As stated, some parts of the example config will need to be edited. Here's a list of changes that should be made: -- `delete_images` - set to false by default, if moving from Gemini data, it may be appropriate to set to true -- `access_key`, `secret_key` - credentials for Echo, ensure these are set -- `images_bucket` - name of the bucket where images are stored (and will be removed if `delete_images` is set to true). An example bucket name would be `og-matt-dev` -- `log_config_path` - path to the API's logs, ensure this is set correctly +The example config contains sensible values to use, but there are a few things to bear in mind: +- `wipe_echo` - this can be good to start from a clean slate but depending on the amount of data in the bucket, it may take a while to delete the data from the bucket. Unless you need to start from an empty bucket, leave this set to `false` +- `launch_api` - when set to `true`, this option will launch an instance of the API within the script, and kill it once ingestion has completed - ensure you have stopped any instances of the API running so there's no address collisions! If set to `false`, the script will use an instance of the API pointing elsewhere +- `ingest_mode` - for development environments, `sequential` is the suggested value. `parallel` is meant for ingestion on a highly-specced machine when ingesting for the dev server +- `file_to_restart_ingestion` - an option used for ingestion onto the dev server. For development environments ingesting data for the tests, leave this blank +- `storage_bucket` - set this to the bucket assigned to you for development. This is where your images and waveforms will be stored +- `access_key`, `secret_key` - ensure these are set to the Echo credentials +- `log_config_path` - ensure this is set correctly for your environment Depending on which options are enabled, the script will do the following things: - Wipe database and Echo From 6ba583d7718325b016ef9cde022a8bb801b8ef37 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 19 Jul 2024 13:33:03 +0000 Subject: [PATCH 19/28] DSEGOG-338 Remove `ssh` section of ingestion script config - I've just realised this is no longer used, so it can be removed --- .github/ci_ingest_echo_config.yml | 3 --- util/realistic_data/config.yml.example | 4 +--- util/realistic_data/ingest/config.py | 6 ------ 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/ci_ingest_echo_config.yml b/.github/ci_ingest_echo_config.yml index 2b9ad5f2..3208befe 100644 --- a/.github/ci_ingest_echo_config.yml +++ b/.github/ci_ingest_echo_config.yml @@ -7,9 +7,6 @@ script_options: # If you want the script to restart ingestion midway through, specify the last # successful file that ingested e.g. data/2023-06-04T1200.h5 file_to_restart_ingestion: "" -ssh: - enabled: false - ssh_connection_url: 127.0.0.1 database: connection_uri: mongodb://localhost:27017/opsgateway remote_experiments_file_path: /tmp/experiments_for_mongoimport.json diff --git a/util/realistic_data/config.yml.example b/util/realistic_data/config.yml.example index 2e24dd5f..fcc4cde0 100644 --- a/util/realistic_data/config.yml.example +++ b/util/realistic_data/config.yml.example @@ -3,13 +3,11 @@ script_options: wipe_echo: false launch_api: true import_users: true + # Either 'sequential' or 'parallel' ingest_mode: sequential # If you want the script to restart ingestion midway through, specify the last # successful file that ingested e.g. data/2023-06-04T1200.h5 file_to_restart_ingestion: "" -ssh: - enabled: false - ssh_connection_url: 127.0.0.1 database: connection_uri: mongodb://localhost:27017/opsgateway remote_experiments_file_path: /tmp/experiments_for_mongoimport.json diff --git a/util/realistic_data/ingest/config.py b/util/realistic_data/ingest/config.py index 79e0e6b3..bd43311d 100644 --- a/util/realistic_data/ingest/config.py +++ b/util/realistic_data/ingest/config.py @@ -21,11 +21,6 @@ class ScriptOptions(BaseModel): file_to_restart_ingestion: Optional[str] -class SSH(BaseModel): - enabled: bool - ssh_connection_url: str - - class Database(BaseModel): connection_uri: str remote_experiments_file_path: str @@ -54,7 +49,6 @@ class API(BaseModel): class IngestEchoDataConfig(BaseModel): script_options: ScriptOptions - ssh: SSH database: Database echo: Echo api: API From 71cc0db9aba023c7acf2f18bc26ab78f863c555c Mon Sep 17 00:00:00 2001 From: kevinphippsstfc Date: Mon, 22 Jul 2024 12:06:40 +0000 Subject: [PATCH 20/28] Change from StreamingResponse to basic fastapi Response --- operationsgateway_api/src/routes/export.py | 11 +++++------ operationsgateway_api/src/routes/images.py | 7 +++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/operationsgateway_api/src/routes/export.py b/operationsgateway_api/src/routes/export.py index d2c3ff94..565158ff 100644 --- a/operationsgateway_api/src/routes/export.py +++ b/operationsgateway_api/src/routes/export.py @@ -2,8 +2,7 @@ import logging from typing import List, Optional -from fastapi import APIRouter, Depends, Query -from fastapi.responses import StreamingResponse +from fastapi import APIRouter, Depends, Query, Response from pydantic import Json import pymongo from typing_extensions import Annotated @@ -153,16 +152,16 @@ async def export_records( if type(file_bytes_to_export) == io.BytesIO: # this is a zip file headers = {"Content-Disposition": f'attachment; filename="{filename}.zip"'} - return StreamingResponse( - file_bytes_to_export, + return Response( + file_bytes_to_export.read(), headers=headers, media_type="application/zip", ) elif type(file_bytes_to_export) == io.StringIO: # this is a csv file headers = {"Content-Disposition": f'attachment; filename="{filename}.csv"'} - return StreamingResponse( - file_bytes_to_export, + return Response( + file_bytes_to_export.read(), headers=headers, media_type="text/plain", ) diff --git a/operationsgateway_api/src/routes/images.py b/operationsgateway_api/src/routes/images.py index 5808e2a0..a433b5ba 100644 --- a/operationsgateway_api/src/routes/images.py +++ b/operationsgateway_api/src/routes/images.py @@ -1,8 +1,7 @@ import logging from typing import Optional -from fastapi import APIRouter, Depends, Path, Query -from fastapi.responses import StreamingResponse +from fastapi import APIRouter, Depends, Path, Query, Response from typing_extensions import Annotated from operationsgateway_api.src.auth.authorisation import authorise_token @@ -80,7 +79,7 @@ async def get_full_image( ) # ensure the "file pointer" is reset image_bytes.seek(0) - return StreamingResponse(image_bytes, media_type="image/png") + return Response(image_bytes.read(), media_type="image/png") @router.get( @@ -125,7 +124,7 @@ async def get_colourbar_image( colourmap_name, ) colourbar_image_bytes.seek(0) - return StreamingResponse(colourbar_image_bytes, media_type="image/png") + return Response(colourbar_image_bytes.read(), media_type="image/png") @router.get( From 147d4c8c953bae0e040c3371e6a0af54d1ac24d8 Mon Sep 17 00:00:00 2001 From: kevinphippsstfc Date: Mon, 22 Jul 2024 12:33:13 +0000 Subject: [PATCH 21/28] Update cryptography due to safety check failure --- poetry.lock | 93 +++++++++++++++++++++++------------------------------ 1 file changed, 41 insertions(+), 52 deletions(-) diff --git a/poetry.lock b/poetry.lock index 92e1cc34..e93c987d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. [[package]] name = "annotated-types" @@ -537,43 +537,43 @@ test = ["python-dateutil"] [[package]] name = "cryptography" -version = "42.0.7" +version = "42.0.8" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, - {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, - {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, - {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, - {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, - {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, - {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, ] [package.dependencies] @@ -1857,8 +1857,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.1" pytz = ">=2020.1" @@ -2524,7 +2524,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2532,16 +2531,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2558,7 +2549,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2566,7 +2556,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2684,24 +2673,24 @@ python-versions = ">=3.6" files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d92f81886165cb14d7b067ef37e142256f1c6a90a65cd156b063a43da1708cfd"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, - {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b5edda50e5e9e15e54a6a8a0070302b00c518a9d32accc2346ad6c984aacd279"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, - {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:7048c338b6c86627afb27faecf418768acb6331fc24cfa56c93e8c9780f815fa"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, - {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, @@ -2709,7 +2698,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, - {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3fcc54cb0c8b811ff66082de1680b4b14cf8a81dce0d4fbf665c2265a81e07a1"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, @@ -2717,7 +2706,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, - {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:665f58bfd29b167039f714c6998178d27ccd83984084c286110ef26b230f259f"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, @@ -2725,7 +2714,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, - {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:9eb5dee2772b0f704ca2e45b1713e4e5198c18f515b52743576d196348f374d3"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, From 6451d59b37b9b3132d2c5485e5bcf94989b8c0b1 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 23 Jul 2024 07:46:28 +0000 Subject: [PATCH 22/28] DSEGOG-309 Change function to return `ChannelManifestModel` - Added this function instead of editing `get_most_recent_manifest()` so I can gradually refactor the codebase - This will eventually become `get_most_recent_manifest()` --- .../src/channels/channel_manifest.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/operationsgateway_api/src/channels/channel_manifest.py b/operationsgateway_api/src/channels/channel_manifest.py index 473a29eb..4a3a6e0a 100644 --- a/operationsgateway_api/src/channels/channel_manifest.py +++ b/operationsgateway_api/src/channels/channel_manifest.py @@ -88,14 +88,32 @@ async def get_most_recent_manifest() -> dict: return manifest_data + # TODO - does it need to be static? + @staticmethod + async def get_most_recent_manifest_new() -> ChannelManifestModel: + """ + Get the most up to date manifest file from MongoDB and return it to the user + TODO + """ + + log.info("Getting most recent channel manifest file") + manifest_data = await MongoDBInterface.find_one( + "channels", + sort=[("_id", pymongo.DESCENDING)], + ) + + try: + return ChannelManifestModel(**manifest_data) + except ValidationError as exc: + raise ModelError(str(exc)) from exc + @staticmethod async def get_channel(channel_name: str) -> ChannelModel: """ Look for the most recent manifest file and return a specific channel's metadata from that file """ - manifest_data = await ChannelManifest.get_most_recent_manifest() - manifest = ChannelManifestModel(**manifest_data) + manifest = await ChannelManifest.get_most_recent_manifest_new() try: return manifest.channels[channel_name] except KeyError as exc: From b372f0a955c9599372e77d8f81046054868557bf Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 23 Jul 2024 09:33:00 +0000 Subject: [PATCH 23/28] DSEGOG-309 Refactor usage of `ChannelManifest` for use of Pydantic models --- .../src/channels/channel_manifest.py | 5 ++--- .../src/records/export_handler.py | 16 +++++++++------- .../src/records/ingestion/channel_checks.py | 14 +++++++++++--- .../src/records/ingestion/hdf_handler.py | 2 +- operationsgateway_api/src/routes/channels.py | 3 ++- operationsgateway_api/src/routes/export.py | 4 ++-- .../src/routes/ingest_data.py | 2 +- test/channels/test_channel_manifest.py | 3 ++- test/channels/test_manifest_validator.py | 11 +++++++---- test/records/ingestion/test_channel.py | 18 +++++++++--------- 10 files changed, 46 insertions(+), 32 deletions(-) diff --git a/operationsgateway_api/src/channels/channel_manifest.py b/operationsgateway_api/src/channels/channel_manifest.py index 4a3a6e0a..fb68a1a6 100644 --- a/operationsgateway_api/src/channels/channel_manifest.py +++ b/operationsgateway_api/src/channels/channel_manifest.py @@ -29,7 +29,6 @@ def __init__(self, manifest_input: SpooledTemporaryFile) -> None: ) from exc manifest_file["_id"] = self._add_id() - self.data = self._use_model(manifest_file) async def insert(self) -> None: @@ -46,13 +45,13 @@ async def validate(self, bypass_channel_check: bool) -> None: Validate the user's incoming manifest file by comparing that with the latest version stored in the database """ - stored_manifest = await ChannelManifest.get_most_recent_manifest() + stored_manifest = await ChannelManifest.get_most_recent_manifest_new() # Validation can only be done if there's an existing manifest file stored if stored_manifest: validator = ManifestValidator( self.data, - self._use_model(stored_manifest), + stored_manifest, bypass_channel_check, ) validator.perform_validation() diff --git a/operationsgateway_api/src/records/export_handler.py b/operationsgateway_api/src/records/export_handler.py index f59d1438..c352c612 100644 --- a/operationsgateway_api/src/records/export_handler.py +++ b/operationsgateway_api/src/records/export_handler.py @@ -5,6 +5,7 @@ from operationsgateway_api.src.config import Config from operationsgateway_api.src.exceptions import ExportError +from operationsgateway_api.src.models import ChannelManifestModel from operationsgateway_api.src.records.image import Image from operationsgateway_api.src.records.waveform import Waveform @@ -18,7 +19,7 @@ class ExportHandler: def __init__( self, records_data: List[dict], - channel_manifest_dict: dict, + channel_manifest: ChannelManifestModel, projection: List[str], lower_level: int, upper_level: int, @@ -32,7 +33,7 @@ def __init__( Store all of the information that needs to be processed during the export """ self.records_data = records_data - self.channel_manifest_dict = channel_manifest_dict + self.channel_manifest = channel_manifest self.projection = projection self.lower_level = lower_level self.upper_level = upper_level @@ -120,9 +121,10 @@ def _create_main_csv_headers(self) -> None: if proj.split(".")[0] == "channels": # image and waveform data will be exported to separate files so will not # have values put in the main csv file - if ( - self.channel_manifest_dict["channels"][channel_name]["type"] - ) not in ["image", "waveform"]: + if (self.channel_manifest.channels[channel_name].type_) not in [ + "image", + "waveform", + ]: line = self._add_value_to_csv_line(line, channel_name) else: # this must be a "metadata" channel @@ -157,11 +159,11 @@ async def _process_data_channel( that will be added to the main CSV file. """ # process an image channel - if self.channel_manifest_dict["channels"][channel_name]["type"] == "image": + if self.channel_manifest.channels[channel_name].type_ == "image": log.info("Channel %s is an image", channel_name) await self._add_image_to_zip(record_data, record_id, channel_name) # process a waveform channel - elif self.channel_manifest_dict["channels"][channel_name]["type"] == "waveform": + elif self.channel_manifest.channels[channel_name].type_ == "waveform": log.info("Channel %s is a waveform", channel_name) x_units = record_data["channels"][channel_name]["metadata"].setdefault( "x_units", diff --git a/operationsgateway_api/src/records/ingestion/channel_checks.py b/operationsgateway_api/src/records/ingestion/channel_checks.py index 292c4411..415b8a15 100644 --- a/operationsgateway_api/src/records/ingestion/channel_checks.py +++ b/operationsgateway_api/src/records/ingestion/channel_checks.py @@ -1,7 +1,10 @@ import logging +from typing import Dict, List import numpy as np +from operationsgateway_api.src.models import ChannelModel + log = logging.getLogger() @@ -32,7 +35,7 @@ def __init__( ] def set_channels(self, manifest) -> None: - self.manifest_channels = manifest["channels"] + self.manifest_channels = manifest.channels def _merge_internal_failed( self, @@ -86,7 +89,7 @@ async def channel_dtype_checks(self): if "channel_dtype" in value: if ( - self.manifest_channels[key]["type"] != value["channel_dtype"] + self.manifest_channels[key].type_ != value["channel_dtype"] or value["channel_dtype"] not in self.supported_channel_types ): rejected_channels.append( @@ -408,7 +411,12 @@ def unrecognised_attribute_checks(self): return rejected_channels - def _check_name(self, rejected_channels, manifest, key): + def _check_name( + self, + rejected_channels: List[dict], + manifest: Dict[str, ChannelModel], + key, + ): """ Checks if the channel name appears in the most recent channel manifest diff --git a/operationsgateway_api/src/records/ingestion/hdf_handler.py b/operationsgateway_api/src/records/ingestion/hdf_handler.py index decef107..ac234db9 100644 --- a/operationsgateway_api/src/records/ingestion/hdf_handler.py +++ b/operationsgateway_api/src/records/ingestion/hdf_handler.py @@ -239,7 +239,7 @@ async def extract_channels(self) -> None: """ internal_failed_channel = [] - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() for channel_name, value in self.hdf_file.items(): channel_metadata = dict(value.attrs) diff --git a/operationsgateway_api/src/routes/channels.py b/operationsgateway_api/src/routes/channels.py index f79deb1d..8995a0ae 100644 --- a/operationsgateway_api/src/routes/channels.py +++ b/operationsgateway_api/src/routes/channels.py @@ -79,7 +79,8 @@ async def get_channels( log.info("Getting channel metadata from database") - return await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() + return manifest.model_dump(by_alias=True, exclude_unset=True) @router.get( diff --git a/operationsgateway_api/src/routes/export.py b/operationsgateway_api/src/routes/export.py index d2c3ff94..2ec284fc 100644 --- a/operationsgateway_api/src/routes/export.py +++ b/operationsgateway_api/src/routes/export.py @@ -130,11 +130,11 @@ async def export_records( if len(records_data) == 0: raise ExportError("No records found to export") - channel_mainfest_dict = await ChannelManifest.get_most_recent_manifest() + channel_mainfest = await ChannelManifest.get_most_recent_manifest_new() export_handler = ExportHandler( records_data, - channel_mainfest_dict, + channel_mainfest, projection, lower_level, upper_level, diff --git a/operationsgateway_api/src/routes/ingest_data.py b/operationsgateway_api/src/routes/ingest_data.py index 00bec144..d442a6dd 100644 --- a/operationsgateway_api/src/routes/ingest_data.py +++ b/operationsgateway_api/src/routes/ingest_data.py @@ -72,7 +72,7 @@ async def submit_hdf( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_dict = await channel_checker.channel_checks() diff --git a/test/channels/test_channel_manifest.py b/test/channels/test_channel_manifest.py index 0770096d..c028deb1 100644 --- a/test/channels/test_channel_manifest.py +++ b/test/channels/test_channel_manifest.py @@ -70,7 +70,8 @@ async def test_validate_no_stored_file(self): instance = get_spooled_file() with patch( - "operationsgateway_api.src.channels.channel_manifest.ChannelManifest.get_most_recent_manifest", + "operationsgateway_api.src.channels.channel_manifest.ChannelManifest" + ".get_most_recent_manifest_new", return_value=None, ): await instance.validate(bypass_channel_check=True) diff --git a/test/channels/test_manifest_validator.py b/test/channels/test_manifest_validator.py index a343dd5f..1eb5cb29 100644 --- a/test/channels/test_manifest_validator.py +++ b/test/channels/test_manifest_validator.py @@ -3,6 +3,7 @@ import pytest from operationsgateway_api.src.exceptions import ChannelManifestError +from operationsgateway_api.src.models import ChannelManifestModel class TestManifestValidator: @@ -31,11 +32,12 @@ async def test_unmatched_channel_field(self, create_manifest_file): }, }, } + altered_manifest = ChannelManifestModel(**altered_content) with patch( "operationsgateway_api.src.channels.channel_manifest.ChannelManifest." - "get_most_recent_manifest", - return_value=altered_content, + "get_most_recent_manifest_new", + return_value=altered_manifest, ): with pytest.raises(ChannelManifestError, match="has been modified on the"): await create_manifest_file.validate(bypass_channel_check=False) @@ -52,10 +54,11 @@ async def test_stored_channel_missing(self, create_manifest_file): }, }, } + altered_manifest = ChannelManifestModel(**altered_content) with patch( "operationsgateway_api.src.channels.channel_manifest.ChannelManifest." - "get_most_recent_manifest", - return_value=altered_content, + "get_most_recent_manifest_new", + return_value=altered_manifest, ): await create_manifest_file.validate(bypass_channel_check=False) diff --git a/test/records/ingestion/test_channel.py b/test/records/ingestion/test_channel.py index 08b66fb3..32deebd5 100644 --- a/test/records/ingestion/test_channel.py +++ b/test/records/ingestion/test_channel.py @@ -70,7 +70,7 @@ async def test_channel_checks_success(self, remove_hdf_file): internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) async_functions = [ channel_checker.channel_dtype_checks, @@ -205,7 +205,7 @@ async def test_channel_dtype_fail(self, remove_hdf_file, altered_channel, respon images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -387,7 +387,7 @@ async def test_required_attribute( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response, extra) @@ -521,7 +521,7 @@ async def test_optional_attribute_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -669,7 +669,7 @@ async def test_dataset_checks_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response, extra) @@ -813,7 +813,7 @@ async def test_unrecognised_attribute_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -893,7 +893,7 @@ async def test_channel_name_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -1018,7 +1018,7 @@ async def test_channel_checks_separate( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response, channels=channels_check) @@ -1096,7 +1096,7 @@ async def test_channel_checks_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest() + manifest = await ChannelManifest.get_most_recent_manifest_new() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) From 4ebede5403adff1825ccf95c484487a3d317b4e9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 23 Jul 2024 09:36:26 +0000 Subject: [PATCH 24/28] DSEGOG-309 Make use of `_use_model()` in `get_most_recent_manifest()` --- operationsgateway_api/src/channels/channel_manifest.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/operationsgateway_api/src/channels/channel_manifest.py b/operationsgateway_api/src/channels/channel_manifest.py index fb68a1a6..59b09021 100644 --- a/operationsgateway_api/src/channels/channel_manifest.py +++ b/operationsgateway_api/src/channels/channel_manifest.py @@ -64,7 +64,8 @@ def _add_id(self) -> str: return datetime.now().strftime(ID_DATETIME_FORMAT) - def _use_model(self, data: dict) -> ChannelManifestModel: + @staticmethod + def _use_model(data: dict) -> ChannelManifestModel: """ Convert dict into Pydantic model, with exception handling wrapped around the code to perform this @@ -87,12 +88,10 @@ async def get_most_recent_manifest() -> dict: return manifest_data - # TODO - does it need to be static? @staticmethod async def get_most_recent_manifest_new() -> ChannelManifestModel: """ Get the most up to date manifest file from MongoDB and return it to the user - TODO """ log.info("Getting most recent channel manifest file") @@ -101,10 +100,7 @@ async def get_most_recent_manifest_new() -> ChannelManifestModel: sort=[("_id", pymongo.DESCENDING)], ) - try: - return ChannelManifestModel(**manifest_data) - except ValidationError as exc: - raise ModelError(str(exc)) from exc + return ChannelManifest._use_model(manifest_data) @staticmethod async def get_channel(channel_name: str) -> ChannelModel: From d5044ccc513da1964d306751150407b12684c2c3 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 23 Jul 2024 09:40:23 +0000 Subject: [PATCH 25/28] DSEGOG-309 Replace old `get_most_recent_manifest()` with new version using Pydantic models --- .../src/channels/channel_manifest.py | 19 +++---------------- .../src/records/ingestion/hdf_handler.py | 2 +- operationsgateway_api/src/routes/channels.py | 2 +- operationsgateway_api/src/routes/export.py | 2 +- .../src/routes/ingest_data.py | 2 +- test/channels/test_channel_manifest.py | 2 +- test/channels/test_manifest_validator.py | 4 ++-- test/records/ingestion/test_channel.py | 18 +++++++++--------- 8 files changed, 19 insertions(+), 32 deletions(-) diff --git a/operationsgateway_api/src/channels/channel_manifest.py b/operationsgateway_api/src/channels/channel_manifest.py index 59b09021..5a2da103 100644 --- a/operationsgateway_api/src/channels/channel_manifest.py +++ b/operationsgateway_api/src/channels/channel_manifest.py @@ -45,7 +45,7 @@ async def validate(self, bypass_channel_check: bool) -> None: Validate the user's incoming manifest file by comparing that with the latest version stored in the database """ - stored_manifest = await ChannelManifest.get_most_recent_manifest_new() + stored_manifest = await ChannelManifest.get_most_recent_manifest() # Validation can only be done if there's an existing manifest file stored if stored_manifest: @@ -76,20 +76,7 @@ def _use_model(data: dict) -> ChannelManifestModel: raise ModelError(str(exc)) from exc @staticmethod - async def get_most_recent_manifest() -> dict: - """ - Get the most up to date manifest file from MongoDB and return it to the user - """ - log.info("Getting most recent channel manifest file") - manifest_data = await MongoDBInterface.find_one( - "channels", - sort=[("_id", pymongo.DESCENDING)], - ) - - return manifest_data - - @staticmethod - async def get_most_recent_manifest_new() -> ChannelManifestModel: + async def get_most_recent_manifest() -> ChannelManifestModel: """ Get the most up to date manifest file from MongoDB and return it to the user """ @@ -108,7 +95,7 @@ async def get_channel(channel_name: str) -> ChannelModel: Look for the most recent manifest file and return a specific channel's metadata from that file """ - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() try: return manifest.channels[channel_name] except KeyError as exc: diff --git a/operationsgateway_api/src/records/ingestion/hdf_handler.py b/operationsgateway_api/src/records/ingestion/hdf_handler.py index ac234db9..decef107 100644 --- a/operationsgateway_api/src/records/ingestion/hdf_handler.py +++ b/operationsgateway_api/src/records/ingestion/hdf_handler.py @@ -239,7 +239,7 @@ async def extract_channels(self) -> None: """ internal_failed_channel = [] - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() for channel_name, value in self.hdf_file.items(): channel_metadata = dict(value.attrs) diff --git a/operationsgateway_api/src/routes/channels.py b/operationsgateway_api/src/routes/channels.py index 8995a0ae..5acd62f9 100644 --- a/operationsgateway_api/src/routes/channels.py +++ b/operationsgateway_api/src/routes/channels.py @@ -79,7 +79,7 @@ async def get_channels( log.info("Getting channel metadata from database") - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() return manifest.model_dump(by_alias=True, exclude_unset=True) diff --git a/operationsgateway_api/src/routes/export.py b/operationsgateway_api/src/routes/export.py index 2ec284fc..53430edd 100644 --- a/operationsgateway_api/src/routes/export.py +++ b/operationsgateway_api/src/routes/export.py @@ -130,7 +130,7 @@ async def export_records( if len(records_data) == 0: raise ExportError("No records found to export") - channel_mainfest = await ChannelManifest.get_most_recent_manifest_new() + channel_mainfest = await ChannelManifest.get_most_recent_manifest() export_handler = ExportHandler( records_data, diff --git a/operationsgateway_api/src/routes/ingest_data.py b/operationsgateway_api/src/routes/ingest_data.py index d442a6dd..00bec144 100644 --- a/operationsgateway_api/src/routes/ingest_data.py +++ b/operationsgateway_api/src/routes/ingest_data.py @@ -72,7 +72,7 @@ async def submit_hdf( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_dict = await channel_checker.channel_checks() diff --git a/test/channels/test_channel_manifest.py b/test/channels/test_channel_manifest.py index c028deb1..235d94f2 100644 --- a/test/channels/test_channel_manifest.py +++ b/test/channels/test_channel_manifest.py @@ -71,7 +71,7 @@ async def test_validate_no_stored_file(self): with patch( "operationsgateway_api.src.channels.channel_manifest.ChannelManifest" - ".get_most_recent_manifest_new", + ".get_most_recent_manifest", return_value=None, ): await instance.validate(bypass_channel_check=True) diff --git a/test/channels/test_manifest_validator.py b/test/channels/test_manifest_validator.py index 1eb5cb29..9c763ca3 100644 --- a/test/channels/test_manifest_validator.py +++ b/test/channels/test_manifest_validator.py @@ -36,7 +36,7 @@ async def test_unmatched_channel_field(self, create_manifest_file): with patch( "operationsgateway_api.src.channels.channel_manifest.ChannelManifest." - "get_most_recent_manifest_new", + "get_most_recent_manifest", return_value=altered_manifest, ): with pytest.raises(ChannelManifestError, match="has been modified on the"): @@ -58,7 +58,7 @@ async def test_stored_channel_missing(self, create_manifest_file): with patch( "operationsgateway_api.src.channels.channel_manifest.ChannelManifest." - "get_most_recent_manifest_new", + "get_most_recent_manifest", return_value=altered_manifest, ): await create_manifest_file.validate(bypass_channel_check=False) diff --git a/test/records/ingestion/test_channel.py b/test/records/ingestion/test_channel.py index 32deebd5..08b66fb3 100644 --- a/test/records/ingestion/test_channel.py +++ b/test/records/ingestion/test_channel.py @@ -70,7 +70,7 @@ async def test_channel_checks_success(self, remove_hdf_file): internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) async_functions = [ channel_checker.channel_dtype_checks, @@ -205,7 +205,7 @@ async def test_channel_dtype_fail(self, remove_hdf_file, altered_channel, respon images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -387,7 +387,7 @@ async def test_required_attribute( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response, extra) @@ -521,7 +521,7 @@ async def test_optional_attribute_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -669,7 +669,7 @@ async def test_dataset_checks_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response, extra) @@ -813,7 +813,7 @@ async def test_unrecognised_attribute_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -893,7 +893,7 @@ async def test_channel_name_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) @@ -1018,7 +1018,7 @@ async def test_channel_checks_separate( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response, channels=channels_check) @@ -1096,7 +1096,7 @@ async def test_channel_checks_fail( images, internal_failed_channel, ) - manifest = await ChannelManifest.get_most_recent_manifest_new() + manifest = await ChannelManifest.get_most_recent_manifest() channel_checker.set_channels(manifest) channel_response = create_channel_response(response) From b860bc0c0c038739f818c857ee289873012f14b4 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 23 Jul 2024 13:57:56 +0000 Subject: [PATCH 26/28] DSEGOG-309 Fix bugs for when there's no manifest file in the database --- .../src/channels/channel_manifest.py | 3 ++- .../src/records/ingestion/channel_checks.py | 6 ++++++ test/channels/test_channel_manifest.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/operationsgateway_api/src/channels/channel_manifest.py b/operationsgateway_api/src/channels/channel_manifest.py index 5a2da103..33bde90a 100644 --- a/operationsgateway_api/src/channels/channel_manifest.py +++ b/operationsgateway_api/src/channels/channel_manifest.py @@ -71,7 +71,8 @@ def _use_model(data: dict) -> ChannelManifestModel: code to perform this """ try: - return ChannelManifestModel(**data) + model = ChannelManifestModel(**data) if data else None + return model except ValidationError as exc: raise ModelError(str(exc)) from exc diff --git a/operationsgateway_api/src/records/ingestion/channel_checks.py b/operationsgateway_api/src/records/ingestion/channel_checks.py index 415b8a15..fdcf7146 100644 --- a/operationsgateway_api/src/records/ingestion/channel_checks.py +++ b/operationsgateway_api/src/records/ingestion/channel_checks.py @@ -3,6 +3,7 @@ import numpy as np +from operationsgateway_api.src.exceptions import ChannelManifestError from operationsgateway_api.src.models import ChannelModel @@ -35,6 +36,11 @@ def __init__( ] def set_channels(self, manifest) -> None: + if not manifest: + raise ChannelManifestError( + "There is no manifest file stored in the database, channel checks" + " against cannot occur unless there is one present", + ) self.manifest_channels = manifest.channels def _merge_internal_failed( diff --git a/test/channels/test_channel_manifest.py b/test/channels/test_channel_manifest.py index 235d94f2..999bc512 100644 --- a/test/channels/test_channel_manifest.py +++ b/test/channels/test_channel_manifest.py @@ -6,6 +6,7 @@ from operationsgateway_api.src.channels.channel_manifest import ChannelManifest from operationsgateway_api.src.exceptions import ChannelManifestError, ModelError +from operationsgateway_api.src.models import ChannelManifestModel from operationsgateway_api.src.mongo.interface import MongoDBInterface @@ -92,3 +93,18 @@ async def test_insert_success(self, remove_manifest_entry): ) expected_manifest = json.loads(success_manifest_content) assert channel_manifest == expected_manifest + + @pytest.mark.parametrize( + "data, expected_return", + [ + pytest.param( + json.loads(success_manifest_content), + ChannelManifestModel(**json.loads(success_manifest_content)), + id="Typical dictionary input", + ), + pytest.param(None, None, id="Empty input"), + ], + ) + def test_use_model(self, data, expected_return): + model = ChannelManifest._use_model(data) + assert model == expected_return From 236d83946b32b65217312fc788e8305129da6694 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 31 Jul 2024 10:38:25 +0000 Subject: [PATCH 27/28] DSEGOG-337 Remove duplication of path variable --- util/realistic_data/daily_ingestor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/util/realistic_data/daily_ingestor.py b/util/realistic_data/daily_ingestor.py index e59279af..d292aa3c 100644 --- a/util/realistic_data/daily_ingestor.py +++ b/util/realistic_data/daily_ingestor.py @@ -36,11 +36,8 @@ def main(): current_datetime = ( datetime.now().replace(microsecond=0).strftime(in_progress_date_format) ) - Path(f"{hdf_data_directory}/in_progress_{current_datetime}").mkdir( - parents=True, - exist_ok=True, - ) in_progress_dir = Path(f"{hdf_data_directory}/in_progress_{current_datetime}") + in_progress_dir.mkdir(parents=True, exist_ok=True) print(f"Created {in_progress_dir} directory") files_to_ingest = [] From 74992688d4bd21ab0f4e52ad18e8689493a84244 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 31 Jul 2024 13:18:30 +0000 Subject: [PATCH 28/28] DSEGOG-269 Add comments to image and waveform thumbnail code & tests --- operationsgateway_api/src/records/waveform.py | 6 +++++- test/images/test_image.py | 21 +++++++++++++++++-- test/records/test_waveform.py | 8 +++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/operationsgateway_api/src/records/waveform.py b/operationsgateway_api/src/records/waveform.py index 3663b002..16c65b10 100644 --- a/operationsgateway_api/src/records/waveform.py +++ b/operationsgateway_api/src/records/waveform.py @@ -96,7 +96,11 @@ def _create_thumbnail_plot(self, buffer) -> None: plt.box(False) # Setting bbox_inches="tight" and pad_inches=0 removes padding around figure - # to make best use of the limited pixels available in a thumbnail + # to make best use of the limited pixels available in a thumbnail. Because of + # this, dpi has been set to 130 to offset the tight bbox removing white space + # around the figure and there keeps the thumbnail size calculation correct. The + # default dpi is 100 but that will result in thumbnails smaller than the + # configuration setting, hence the value of 130 plt.savefig(buffer, format="PNG", bbox_inches="tight", pad_inches=0, dpi=130) # Flushes the plot to remove data from previously ingested waveforms plt.clf() diff --git a/test/images/test_image.py b/test/images/test_image.py index 4fc73855..dc9d3c20 100644 --- a/test/images/test_image.py +++ b/test/images/test_image.py @@ -64,15 +64,32 @@ def test_create_thumbnail(self): # a purely black 300x300 square created in the test_image above @pytest.mark.parametrize( + # image_size parameter = (rows, columns), not (width, height) which is what + # the other parameters in this test use. image_size is the shape of the numpy + # array. See https://data-flair.training/blogs/numpy-broadcasting/ for a good + # illustration "image_size, config_thumbnail_size, expected_thumbnail_size", [ pytest.param((300, 300), (50, 50), (50, 50), id="50x50 thumbnail"), - pytest.param((400, 300), (60, 80), (60, 80), id="60x80 thumbnail"), + # (400, 300) makes a portrait image, as per numpy array shape (see above + # comment) + pytest.param( + (400, 300), + (60, 80), + (60, 80), + id="60x80 thumbnail (portrait)", + ), + pytest.param( + (300, 400), + (60, 80), + (60, 45), + id="60x45 thumbnail (landscape)", + ), pytest.param( (300, 300), (75, 100), (75, 75), - id="75x100 thumbnail (square image)", + id="75x75 thumbnail (square image)", ), ], ) diff --git a/test/records/test_waveform.py b/test/records/test_waveform.py index f65cc880..eca49ed1 100644 --- a/test/records/test_waveform.py +++ b/test/records/test_waveform.py @@ -34,10 +34,10 @@ async def test_waveform_not_found(self): @pytest.mark.parametrize( "config_thumbnail_size", [ - pytest.param((50, 50), id="50x50 thumbnail"), - pytest.param((60, 80), id="60x80 thumbnail"), - pytest.param((90, 40), id="90x40 thumbnail"), - pytest.param((75, 100), id="75x100 thumbnail"), + pytest.param((50, 50), id="50x50 thumbnail (square)"), + pytest.param((60, 80), id="60x80 thumbnail (portrait)"), + pytest.param((90, 40), id="90x40 thumbnail (landscape)"), + pytest.param((75, 100), id="75x100 thumbnail (portrait)"), ], ) def test_create_thumbnail_plot_size(self, config_thumbnail_size):