Added toolkits for common microservices, upgraded build to use 'uv'

lib/workload/components/dynamodb-partitioned-table/index.ts

@@ -27,7 +27,9 @@ export class DynamodbPartitionedPipelineConstruct extends Construct {
       },
       tableName: props.tableName,
       removalPolicy: props.removalPolicy || RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE,
-      pointInTimeRecovery: true,
+      pointInTimeRecoverySpecification: {
+        pointInTimeRecoveryEnabled: true,
+      },
     });
 
     // Set outputs

lib/workload/components/python-fastq-tools-layer/fastq_tools_layer/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "fastq_tools"
+version = "0.0.1"
+description = "Workflow Manager Lambda Layers"
+license = "GPL-3.0-or-later"
+authors = [
+    "Alexis Lucattini"
+]
+homepage = "https://github.com/umccr/orcabus"
+repository = "https://github.com/umccr/orcabus"
+
+[tool.poetry.dependencies]
+python = "^3.12, <3.13"
+requests = "^2.32.3"
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.0.0"  # For testing only
+# For typehinting only, not required at runtime
+mypy-boto3-ssm = "^1.34"
+mypy-boto3-secretsmanager = "^1.34"

lib/workload/components/python-fastq-tools-layer/fastq_tools_layer/src/fastq_tools/__init__.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""
+Fastq tools to be used by various lambdas as needed
+"""
+from .utils.models import(
+    FastqListRow,
+    FastqListRowCreate,
+    FastqStorageObject,
+    FileStorageObject,
+    FastqSet,
+    FastqSetCreate,
+    Job,
+    JobStatus,
+    JobType
+)
+
+from .utils.query_helpers import (
+    get_fastq,
+    get_fastqs,
+    get_fastqs_in_instrument_run_id,
+    get_fastqs_in_library,
+    get_fastqs_in_library_list,
+    get_fastqs_in_libraries_and_instrument_run_id,
+    get_fastqs_in_sample,
+    get_fastqs_in_subject,
+    get_fastqs_in_individual,
+    get_fastqs_in_project,
+    get_fastq_set,
+    get_fastq_jobs
+)
+
+
+from .utils.update_helpers import (
+    add_qc_stats,
+    add_read_count,
+    add_file_compression_information,
+    add_ntsm_storage_object,
+    add_read_set,
+    detach_read_set,
+    validate_fastq,
+    invalidate_fastq
+)
+
+from .utils.create_helpers import (
+    create_fastq_set_object,
+    create_fastq_list_row_object,
+)
+
+from .utils.workflow_helpers import (
+    to_cwl
+)
+
+from .utils.job_helpers import (
+    run_qc_stats,
+    run_ntsm,
+    run_file_compression_stats
+)
+
+
+__all__ = [
+    # Models
+    "FastqListRow",
+    "FastqListRowCreate",
+    "FastqStorageObject",
+    "FileStorageObject",
+    "FastqSet",
+    "FastqSetCreate",
+    "Job",
+    "JobStatus",
+    "JobType",
+
+    # Query helpers
+    "get_fastq",
+    "get_fastqs",
+    "get_fastqs_in_instrument_run_id",
+    "get_fastqs_in_library",
+    "get_fastqs_in_library_list",
+    "get_fastqs_in_libraries_and_instrument_run_id",
+    "get_fastqs_in_sample",
+    "get_fastqs_in_subject",
+    "get_fastqs_in_individual",
+    "get_fastqs_in_project",
+
+    # Fastq Set Query helpers
+    "get_fastq_set",
+
+    # Job helpers
+    "get_fastq_jobs",
+
+    # Create helpers,
+    "create_fastq_list_row_object",
+    "create_fastq_set_object",
+
+    # Update helpers
+    "add_qc_stats",
+    "add_read_count",
+    "add_file_compression_information",
+    "add_ntsm_storage_object",
+    "add_read_set",
+    "detach_read_set",
+    "validate_fastq",
+    "invalidate_fastq",
+
+    # Workflow helpers
+    "to_cwl",
+
+    # Job helpers
+    "run_qc_stats",
+    "run_ntsm",
+    "run_file_compression_stats"
+]

lib/workload/components/python-fastq-tools-layer/fastq_tools_layer/src/fastq_tools/utils/aws_helpers.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# Standard imports
+import typing
+import boto3
+import json
+from os import environ
+
+
+# Type hinting
+if typing.TYPE_CHECKING:
+    from mypy_boto3_secretsmanager import SecretsManagerClient
+    from mypy_boto3_ssm import SSMClient
+
+
+def get_secretsmanager_client() -> 'SecretsManagerClient':
+    return boto3.client('secretsmanager')
+
+
+def get_ssm_client() -> 'SSMClient':
+    return boto3.client('ssm')
+
+
+def get_secret_value(secret_id) -> str:
+    """
+    Collect the secret value
+    :param secret_id:
+    :return:
+    """
+    # Get the boto3 response
+    get_secret_value_response = get_secretsmanager_client().get_secret_value(SecretId=secret_id)
+
+    return get_secret_value_response['SecretString']
+
+
+def get_ssm_value(parameter_name) -> str:
+    # Get the boto3 response
+    get_ssm_parameter_response = get_ssm_client().get_parameter(Name=parameter_name)
+
+    return get_ssm_parameter_response['Parameter']['Value']
+
+
+def get_orcabus_token() -> str:
+    """
+    From the AWS Secrets Manager, retrieve the OrcaBus token.
+    :return:
+    """
+    return json.loads(get_secret_value(environ.get("ORCABUS_TOKEN_SECRET_ID")))['id_token']
+
+
+def get_hostname() -> str:
+    return get_ssm_value(environ.get("HOSTNAME_SSM_PARAMETER"))

lib/workload/components/python-fastq-tools-layer/fastq_tools_layer/src/fastq_tools/utils/create_helpers.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+"""
+Update helpers for the update script.
+
+- add_qc_stats
+- add_read_count
+- add_ntsm_storage_object / add_ntsm
+- add_fastq_pair_storage_object / add_read_set
+- detach_fastq_pair_storage_object / detach_read_set
+- validate
+- invalidate
+"""
+
+# Standard imports
+
+# Local imports
+from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
+from .request_helpers import post_request
+from .models import FastqListRow, FastqSet, FastqListRowCreate, FastqSetCreate
+
+
+def create_fastq_list_row_object(fastq_list_row_create_dict: FastqListRowCreate) -> FastqListRow:
+    """
+    Add a fastq list row object to the database.
+    Returns the created fastq list row object
+    """
+    return post_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}",
+        params=fastq_list_row_create_dict
+    )
+
+
+def create_fastq_set_object(fastq_set_create_dict: FastqSetCreate) -> FastqSet:
+    """
+    Add a fastq list row object to the database.
+    Returns the created fastq list row object
+    """
+    return post_request(
+        f"{FASTQ_SET_ENDPOINT}",
+        params=fastq_set_create_dict
+    )

lib/workload/components/python-fastq-tools-layer/fastq_tools_layer/src/fastq_tools/utils/globals.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+import re
+
+# AWS PARAMETERS
+FASTQ_SUBDOMAIN_NAME = "fastq"
+
+# API ENDPOINTS
+FASTQ_LIST_ROW_ENDPOINT = "api/v1/fastq"
+FASTQ_SET_ENDPOINT = "api/v1/fastqSet"
+
+# REGEX
+ORCABUS_ULID_REGEX_MATCH = re.compile(r'^[a-z0-9]{3}\.[A-Z0-9]{26}$')

lib/workload/components/python-fastq-tools-layer/fastq_tools_layer/src/fastq_tools/utils/job_helpers.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+"""
+Update helpers for the update script.
+
+- run_qc_stats
+- run_ntsm
+- run_file_compression_information
+"""
+
+# Standard imports
+
+# Local imports
+from .globals import FASTQ_LIST_ROW_ENDPOINT
+from .request_helpers import patch_request
+from .models import Job
+
+
+def run_qc_stats(fastq_id) -> Job:
+    """
+    Add QC stats to a fastq_id.
+
+    :param fastq_id: Fastq str
+    """
+    return patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}:runQcStats",
+    )
+
+
+def run_ntsm(fastq_id) -> Job:
+    """
+    Run ntsm for a fastq_id.
+
+    :param fastq_id: Fastq str
+    """
+    return patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}:runNtsm",
+    )
+
+
+def run_file_compression_stats(fastq_id) -> Job:
+    """
+    Run file compression stats for a fastq_id.
+
+    :param fastq_id: Fastq str
+    """
+    return patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}:runFileCompressionInformation",
+    )
+